unchecked_malloc() for Skia on OSX.

On tcmalloc-based versions of Chromium, skia resets the global new
handler to prevent the oom-killer from kicking in.  Traditionally the
OSX developers have been agin this, because it's not thread-safe, and
we have questions about whether it's even the right thing to do.  This
change should be thread-safe because g_old_malloc is only set on the
main thread at startup.

As far as whether it's the right thing to do, I'm tired and want to
quit hearing about it.

Additionally, remove a bit of leftover pre-10.6 code.

BUG=103980,73751,117949


Review URL: https://chromiumcodereview.appspot.com/10908245

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@157646 0039d316-1c4b-4281-b951-d872f2087c98

4 files changed, 101 insertions, 82 deletions

diff --git a/base/process_util.h b/base/process_util.h
index cf358dc..3fb7e2ba 100644
--- a/base/process_util.h
+++ b/base/process_util.h
@@ -842,11 +842,6 @@ BASE_EXPORT void EnableTerminationOnHeapCorruption();
 // Turns on process termination if memory runs out.
 BASE_EXPORT void EnableTerminationOnOutOfMemory();
 
-#if defined(OS_MACOSX)
-// Exposed for testing.
-BASE_EXPORT malloc_zone_t* GetPurgeableZone();
-#endif  // defined(OS_MACOSX)
-
 // Enables stack dump to console output on exception and signals.
 // When enabled, the process will quit immediately. This is meant to be used in
 // unit_tests only! This is not thread-safe: only call from main thread.
@@ -867,6 +862,20 @@ BASE_EXPORT void RaiseProcessToHighPriority();
 void RestoreDefaultExceptionHandler();
 #endif  // defined(OS_MACOSX)
 
+#if defined(OS_MACOSX)
+// Very large images or svg canvases can cause huge mallocs.  Skia
+// does tricks on tcmalloc-based systems to allow malloc to fail with
+// a NULL rather than hit the oom crasher.  This replicates that for
+// OSX.
+//
+// IF YOU USE THIS WITHOUT CONSULTING YOUR FRIENDLY OSX DEVELOPER,
+// YOUR CODE IS LIKELY TO BE REVERTED.  THANK YOU.
+//
+// TODO(shess): Weird place to put it, but this is where the OOM
+// killer currently lives.
+BASE_EXPORT void* UncheckedMalloc(size_t size);
+#endif  // defined(OS_MACOSX)
+
 }  // namespace base
 
 #endif  // BASE_PROCESS_UTIL_H_
diff --git a/base/process_util_mac.mm b/base/process_util_mac.mm
index b257021..4885e4b 100644
--- a/base/process_util_mac.mm
+++ b/base/process_util_mac.mm
@@ -13,7 +13,6 @@
 #include <mach/mach_vm.h>
 #include <mach/shared_region.h>
 #include <mach/task.h>
-#include <mach-o/dyld.h>
 #include <mach-o/nlist.h>
 #include <malloc/malloc.h>
 #import <objc/runtime.h>
@@ -32,12 +31,12 @@
 #include "base/eintr_wrapper.h"
 #include "base/file_util.h"
 #include "base/hash_tables.h"
+#include "base/lazy_instance.h"
 #include "base/logging.h"
 #include "base/mac/mac_util.h"
 #include "base/string_util.h"
 #include "base/sys_info.h"
-#include "base/sys_string_conversions.h"
-#include "base/time.h"
+#include "base/threading/thread_local.h"
 #include "third_party/apple_apsl/CFBase.h"
 #include "third_party/apple_apsl/malloc.h"
 #include "third_party/mach_override/mach_override.h"
@@ -566,20 +565,53 @@ class ScopedClearErrno {
   DISALLOW_COPY_AND_ASSIGN(ScopedClearErrno);
 };
 
+// Combines ThreadLocalBoolean with AutoReset.  It would be convenient
+// to compose ThreadLocalPointer<bool> with AutoReset<bool>, but that
+// would require allocating some storage for the bool.
+class ThreadLocalBooleanAutoReset {
+ public:
+  ThreadLocalBooleanAutoReset(ThreadLocalBoolean* tlb, bool new_value)
+      : scoped_tlb_(tlb),
+        original_value_(tlb->Get()) {
+    scoped_tlb_->Set(new_value);
+  }
+  ~ThreadLocalBooleanAutoReset() {
+    scoped_tlb_->Set(original_value_);
+  }
+
+ private:
+  ThreadLocalBoolean* scoped_tlb_;
+  bool original_value_;
+
+  DISALLOW_COPY_AND_ASSIGN(ThreadLocalBooleanAutoReset);
+};
+
+base::LazyInstance<ThreadLocalBoolean>::Leaky
+    g_unchecked_malloc = LAZY_INSTANCE_INITIALIZER;
+
 void CrMallocErrorBreak() {
   g_original_malloc_error_break();
 
   // Out of memory is certainly not heap corruption, and not necessarily
   // something for which the process should be terminated. Leave that decision
-  // to the OOM killer.
-  if (errno == ENOMEM)
+  // to the OOM killer.  The EBADF case comes up because the malloc library
+  // attempts to log to ASL (syslog) before calling this code, which fails
+  // accessing a Unix-domain socket because of sandboxing.
+  if (errno == ENOMEM || (errno == EBADF && g_unchecked_malloc.Get().Get()))
     return;
 
   // A unit test checks this error message, so it needs to be in release builds.
-  LOG(ERROR) <<
+  PLOG(ERROR) <<
       "Terminating process due to a potential for future heap corruption";
-  int* volatile death_ptr = NULL;
-  *death_ptr = 0xf00bad;
+
+  // Crash by writing to NULL+errno to allow analyzing errno from
+  // crash dump info (setting a breakpad key would re-enter the malloc
+  // library).  Max documented errno in intro(2) is actually 102, but
+  // it really just needs to be "small" to stay on the right vm page.
+  const int kMaxErrno = 256;
+  char* volatile death_ptr = NULL;
+  death_ptr += std::min(errno, kMaxErrno);
+  *death_ptr = '!';
 }
 
 }  // namespace
@@ -836,16 +868,13 @@ id oom_killer_allocWithZone(id self, SEL _cmd, NSZone* zone)
 
 }  // namespace
 
-malloc_zone_t* GetPurgeableZone() {
-  // malloc_default_purgeable_zone only exists on >= 10.6. Use dlsym to grab it
-  // at runtime because it may not be present in the SDK used for compilation.
-  typedef malloc_zone_t* (*malloc_default_purgeable_zone_t)(void);
-  malloc_default_purgeable_zone_t malloc_purgeable_zone =
-      reinterpret_cast<malloc_default_purgeable_zone_t>(
-          dlsym(RTLD_DEFAULT, "malloc_default_purgeable_zone"));
-  if (malloc_purgeable_zone)
-    return malloc_purgeable_zone();
-  return NULL;
+void* UncheckedMalloc(size_t size) {
+  if (g_old_malloc) {
+    ScopedClearErrno clear_errno;
+    ThreadLocalBooleanAutoReset flag(g_unchecked_malloc.Pointer(), true);
+    return g_old_malloc(malloc_default_zone(), size);
+  }
+  return malloc(size);
 }
 
 void EnableTerminationOnOutOfMemory() {
@@ -880,7 +909,7 @@ void EnableTerminationOnOutOfMemory() {
   ChromeMallocZone* default_zone =
       reinterpret_cast<ChromeMallocZone*>(malloc_default_zone());
   ChromeMallocZone* purgeable_zone =
-      reinterpret_cast<ChromeMallocZone*>(GetPurgeableZone());
+      reinterpret_cast<ChromeMallocZone*>(malloc_default_purgeable_zone());
 
   vm_address_t page_start_default = 0;
   vm_address_t page_start_purgeable = 0;
diff --git a/base/process_util_unittest.cc b/base/process_util_unittest.cc
index 6eccb90..7840ce3 100644
--- a/base/process_util_unittest.cc
+++ b/base/process_util_unittest.cc
@@ -1092,83 +1092,58 @@ TEST_F(OutOfMemoryDeathTest, ViaSharedLibraries) {
 // Android doesn't implement posix_memalign().
 #if defined(OS_POSIX) && !defined(OS_ANDROID)
 TEST_F(OutOfMemoryDeathTest, Posix_memalign) {
-  typedef int (*memalign_t)(void **, size_t, size_t);
-#if defined(OS_MACOSX)
-  // posix_memalign only exists on >= 10.6. Use dlsym to grab it at runtime
-  // because it may not be present in the SDK used for compilation.
-  memalign_t memalign =
-      reinterpret_cast<memalign_t>(dlsym(RTLD_DEFAULT, "posix_memalign"));
-#else
-  memalign_t memalign = posix_memalign;
-#endif  // defined(OS_MACOSX)
-  if (memalign) {
-    // Grab the return value of posix_memalign to silence a compiler warning
-    // about unused return values. We don't actually care about the return
-    // value, since we're asserting death.
-    ASSERT_DEATH({
-        SetUpInDeathAssert();
-        EXPECT_EQ(ENOMEM, memalign(&value_, 8, test_size_));
-      }, "");
-  }
+  // Grab the return value of posix_memalign to silence a compiler warning
+  // about unused return values. We don't actually care about the return
+  // value, since we're asserting death.
+  ASSERT_DEATH({
+      SetUpInDeathAssert();
+      EXPECT_EQ(ENOMEM, posix_memalign(&value_, 8, test_size_));
+    }, "");
 }
 #endif  // defined(OS_POSIX) && !defined(OS_ANDROID)
 
 #if defined(OS_MACOSX)
 
-// Purgeable zone tests (if it exists)
+// Purgeable zone tests
 
 TEST_F(OutOfMemoryDeathTest, MallocPurgeable) {
-  malloc_zone_t* zone = base::GetPurgeableZone();
-  if (zone)
-    ASSERT_DEATH({
-        SetUpInDeathAssert();
-        value_ = malloc_zone_malloc(zone, test_size_);
-      }, "");
+  malloc_zone_t* zone = malloc_default_purgeable_zone();
+  ASSERT_DEATH({
+      SetUpInDeathAssert();
+      value_ = malloc_zone_malloc(zone, test_size_);
+    }, "");
 }
 
 TEST_F(OutOfMemoryDeathTest, ReallocPurgeable) {
-  malloc_zone_t* zone = base::GetPurgeableZone();
-  if (zone)
-    ASSERT_DEATH({
-        SetUpInDeathAssert();
-        value_ = malloc_zone_realloc(zone, NULL, test_size_);
-      }, "");
+  malloc_zone_t* zone = malloc_default_purgeable_zone();
+  ASSERT_DEATH({
+      SetUpInDeathAssert();
+      value_ = malloc_zone_realloc(zone, NULL, test_size_);
+    }, "");
 }
 
 TEST_F(OutOfMemoryDeathTest, CallocPurgeable) {
-  malloc_zone_t* zone = base::GetPurgeableZone();
-  if (zone)
-    ASSERT_DEATH({
-        SetUpInDeathAssert();
-        value_ = malloc_zone_calloc(zone, 1024, test_size_ / 1024L);
-      }, "");
+  malloc_zone_t* zone = malloc_default_purgeable_zone();
+  ASSERT_DEATH({
+      SetUpInDeathAssert();
+      value_ = malloc_zone_calloc(zone, 1024, test_size_ / 1024L);
+    }, "");
 }
 
 TEST_F(OutOfMemoryDeathTest, VallocPurgeable) {
-  malloc_zone_t* zone = base::GetPurgeableZone();
-  if (zone)
-    ASSERT_DEATH({
-        SetUpInDeathAssert();
-        value_ = malloc_zone_valloc(zone, test_size_);
-      }, "");
+  malloc_zone_t* zone = malloc_default_purgeable_zone();
+  ASSERT_DEATH({
+      SetUpInDeathAssert();
+      value_ = malloc_zone_valloc(zone, test_size_);
+    }, "");
 }
 
 TEST_F(OutOfMemoryDeathTest, PosixMemalignPurgeable) {
-  malloc_zone_t* zone = base::GetPurgeableZone();
-
-  typedef void* (*zone_memalign_t)(malloc_zone_t*, size_t, size_t);
-  // malloc_zone_memalign only exists on >= 10.6. Use dlsym to grab it at
-  // runtime because it may not be present in the SDK used for compilation.
-  zone_memalign_t zone_memalign =
-      reinterpret_cast<zone_memalign_t>(
-        dlsym(RTLD_DEFAULT, "malloc_zone_memalign"));
-
-  if (zone && zone_memalign) {
-    ASSERT_DEATH({
-        SetUpInDeathAssert();
-        value_ = zone_memalign(zone, 8, test_size_);
-      }, "");
-  }
+  malloc_zone_t* zone = malloc_default_purgeable_zone();
+  ASSERT_DEATH({
+      SetUpInDeathAssert();
+      value_ = malloc_zone_memalign(zone, 8, test_size_);
+    }, "");
 }
 
 // Since these allocation functions take a signed size, it's possible that
diff --git a/skia/ext/SkMemory_new_handler.cpp b/skia/ext/SkMemory_new_handler.cpp
index 5f958c6..2ab49e3 100644
--- a/skia/ext/SkMemory_new_handler.cpp
+++ b/skia/ext/SkMemory_new_handler.cpp
@@ -6,6 +6,8 @@
 #include <stdlib.h>
 #include <new>
 
+#include "base/process_util.h"
+
 #include "third_party/skia/include/core/SkTypes.h"
 #include "third_party/skia/include/core/SkThread.h"
 
@@ -54,10 +56,14 @@ void* sk_malloc_flags(size_t size, unsigned flags) {
     p = malloc(size);
 #else
     if (!(flags & SK_MALLOC_THROW)) {
+#if defined(OS_MACOSX)
+      p = base::UncheckedMalloc(size);
+#else
       SkAutoMutexAcquire lock(gSkNewHandlerMutex);
       std::new_handler old_handler = std::set_new_handler(NULL);
       p = malloc(size);
       std::set_new_handler(old_handler);
+#endif
     } else {
       p = malloc(size);
     }