Automatically adapt to faster/slower uploads of renderer histograms

This replaces the current time based approach (chrome is given N seconds
to upload all renederer histograms) with an asynch callback approach
that waits until all renderers have responded (with their updates). It
uses a fall-back timer to ensure that a hung renderer won't delay things
forever as well.

This causes faster (and complete) updates in about:histograms as well
as generally assuring complete updates during UMA gatherings.

This code was contributed by Raman Tenneti in CL 42496
http://codereview.chromium.org/42496

bug=12850

r=raman
Review URL: http://codereview.chromium.org/113473

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@17123 0039d316-1c4b-4281-b951-d872f2087c98

17 files changed, 525 insertions, 79 deletions

diff --git a/base/histogram.cc b/base/histogram.cc
index 687a549..f50571d 100644
--- a/base/histogram.cc
+++ b/base/histogram.cc
@@ -364,16 +364,6 @@ std::string Histogram::SerializeHistogramInfo(const Histogram& histogram,
 }
 
 // static
-void Histogram::DeserializeHistogramList(
-    const std::vector<std::string>& histograms) {
-  for (std::vector<std::string>::const_iterator it = histograms.begin();
-       it < histograms.end();
-       ++it) {
-    DeserializeHistogramInfo(*it);
-  }
-}
-
-// static
 bool Histogram::DeserializeHistogramInfo(const std::string& histogram_info) {
   if (histogram_info.empty()) {
       return false;
diff --git a/base/histogram.h b/base/histogram.h
index 59981ea..3a3ddb1 100644
--- a/base/histogram.h
+++ b/base/histogram.h
@@ -298,9 +298,7 @@ class Histogram {
   // The following method accepts a list of pickled histograms and
   // builds a histogram and updates shadow copy of histogram data in the
   // browser process.
-  static void DeserializeHistogramList(
-      const std::vector<std::string>& histograms);
-  static bool DeserializeHistogramInfo(const std::string& state);
+  static bool DeserializeHistogramInfo(const std::string& histogram_info);
 
 
   //----------------------------------------------------------------------------
diff --git a/chrome/browser/browser_about_handler.cc b/chrome/browser/browser_about_handler.cc
index 6c46f18..f699b1e 100644
--- a/chrome/browser/browser_about_handler.cc
+++ b/chrome/browser/browser_about_handler.cc
@@ -24,6 +24,7 @@
 #include "chrome/browser/net/dns_global.h"
 #include "chrome/browser/renderer_host/render_process_host.h"
 #include "chrome/browser/renderer_host/render_view_host.h"
+#include "chrome/common/histogram_synchronizer.h"
 #include "chrome/common/jstemplate_builder.h"
 #include "chrome/common/pref_names.h"
 #include "chrome/common/pref_service.h"
@@ -45,6 +46,9 @@
 #include "chrome/browser/views/about_network_dialog.h"
 #endif
 
+using base::Time;
+using base::TimeDelta;
+
 namespace {
 
 // The paths used for the about pages.
@@ -137,16 +141,14 @@ std::string AboutDns() {
 }
 
 std::string AboutHistograms(const std::string& query) {
-  std::string data;
-  for (RenderProcessHost::iterator it = RenderProcessHost::begin();
-       it != RenderProcessHost::end(); ++it) {
-    it->second->Send(new ViewMsg_GetRendererHistograms());
-  }
+  TimeDelta wait_time = TimeDelta::FromMilliseconds(10000);
 
-  // TODO(raman): Delay page layout until we get respnoses
-  // back from renderers, and not have to use a fixed size delay.
-  PlatformThread::Sleep(1000);
+  HistogramSynchronizer* current_synchronizer =
+      HistogramSynchronizer::CurrentSynchronizer();
+  DCHECK(current_synchronizer != NULL);
+  current_synchronizer->FetchRendererHistogramsSynchronously(wait_time);
 
+  std::string data;
   StatisticsRecorder::WriteHTMLGraph(query, &data);
   return data;
 }
diff --git a/chrome/browser/browser_main.cc b/chrome/browser/browser_main.cc
index dd0e825..19b505a 100644
--- a/chrome/browser/browser_main.cc
+++ b/chrome/browser/browser_main.cc
@@ -41,6 +41,7 @@
 #include "chrome/common/chrome_constants.h"
 #include "chrome/common/chrome_paths.h"
 #include "chrome/common/chrome_switches.h"
+#include "chrome/common/histogram_synchronizer.h"
 #include "chrome/common/jstemplate_builder.h"
 #include "chrome/common/main_function_params.h"
 #include "chrome/common/pref_names.h"
@@ -427,6 +428,13 @@ int BrowserMain(const MainFunctionParams& parameters) {
   // Initialize histogram statistics gathering system.
   StatisticsRecorder statistics;
 
+  // Initialize histogram synchronizer system. This is a singleton and is used
+  // for posting tasks via NewRunnableMethod. Its deleted when it goes out of
+  // scope. Even though NewRunnableMethod does AddRef and Release, the object
+  // will not be deleted after the Task is executed.
+  scoped_refptr<HistogramSynchronizer> histogram_synchronizer =
+      new HistogramSynchronizer();
+
   // Initialize the shared instance of user data manager.
   scoped_ptr<UserDataManager> user_data_manager(UserDataManager::Create());
 
@@ -565,14 +573,18 @@ int BrowserMain(const MainFunctionParams& parameters) {
   // Set up a field trial to see what disabling DNS pre-resolution does to
   // latency of network transactions.
   FieldTrial::Probability kDIVISOR = 100;
-  FieldTrial::Probability k_PROBABILITY_PER_GROUP = 10;  // 10%.
+  FieldTrial::Probability k_PROBABILITY_PER_GROUP = 10;  // 10% probability.
+  // For options we don't (currently) wish to test, we use zero probability.
+  FieldTrial::Probability k_PROBABILITY_DISABLED = 0;
   scoped_refptr<FieldTrial> dns_trial = new FieldTrial("DnsImpact", kDIVISOR);
 
   dns_trial->AppendGroup("_disabled_prefetch", k_PROBABILITY_PER_GROUP);
+  // Don't discard names (erase these lines) yet, as we may use them, and we
+  // have histogram data named for these options.
   int disabled_plus_4_connections = dns_trial->AppendGroup(
-      "_disabled_prefetch_4_connections", k_PROBABILITY_PER_GROUP);
+      "_disabled_prefetch_4_connections", k_PROBABILITY_DISABLED);
   int enabled_plus_4_connections = dns_trial->AppendGroup(
-      "_enabled_prefetch_4_connections", k_PROBABILITY_PER_GROUP);
+      "_enabled_prefetch_4_connections", k_PROBABILITY_DISABLED);
 
   scoped_ptr<chrome_browser_net::DnsPrefetcherInit> dns_prefetch_init;
   if (dns_trial->group() == FieldTrial::kNotParticipating ||
diff --git a/chrome/browser/metrics/metrics_service.cc b/chrome/browser/metrics/metrics_service.cc
index 0289dd8..c822500 100644
--- a/chrome/browser/metrics/metrics_service.cc
+++ b/chrome/browser/metrics/metrics_service.cc
@@ -181,6 +181,7 @@
 #include "chrome/browser/search_engines/template_url_model.h"
 #include "chrome/common/child_process_info.h"
 #include "chrome/common/chrome_paths.h"
+#include "chrome/common/histogram_synchronizer.h"
 #include "chrome/common/libxml_utils.h"
 #include "chrome/common/notification_service.h"
 #include "chrome/common/pref_names.h"
@@ -209,6 +210,10 @@ static const char kMetricsType[] = "application/vnd.mozilla.metrics.bz2";
 // The delay, in seconds, after startup before sending the first log message.
 static const int kInitialInterlogDuration = 60;  // one minute
 
+// This specifies the amount of time to wait for all renderers to send their
+// data.
+static const int kMaxHistogramGatheringWaitDuration = 60000;  // 60 seconds.
+
 // The default maximum number of events in a log uploaded to the UMA server.
 static const int kInitialEventLimit = 2400;
 
@@ -881,15 +886,51 @@ void MetricsService::StartLogTransmissionTimer() {
 
   // Right before the UMA transmission gets started, there's one more thing we'd
   // like to record: the histogram of memory usage, so we spawn a task to
-  // collect the memory details and when that task is finished, we arrange for
-  // TryToStartTransmission to take over.
+  // collect the memory details and when that task is finished, it will call
+  // OnMemoryDetailCollectionDone, which will call HistogramSynchronization to
+  // collect histograms from all renderers and then we will call
+  // OnHistogramSynchronizationDone to continue processing.
   MessageLoop::current()->PostDelayedTask(FROM_HERE,
       log_sender_factory_.
-          NewRunnableMethod(&MetricsService::CollectMemoryDetails),
+          NewRunnableMethod(&MetricsService::LogTransmissionTimerDone),
       static_cast<int>(interlog_duration_.InMilliseconds()));
 }
 
-void MetricsService::TryToStartTransmission() {
+void MetricsService::LogTransmissionTimerDone() {
+  Task* task = log_sender_factory_.
+      NewRunnableMethod(&MetricsService::OnMemoryDetailCollectionDone);
+
+  MetricsMemoryDetails* details = new MetricsMemoryDetails(task);
+  details->StartFetch();
+
+  // Collect WebCore cache information to put into a histogram.
+  for (RenderProcessHost::iterator it = RenderProcessHost::begin();
+       it != RenderProcessHost::end(); ++it) {
+    it->second->Send(new ViewMsg_GetCacheResourceStats());
+  }
+}
+
+void MetricsService::OnMemoryDetailCollectionDone() {
+  DCHECK(IsSingleThreaded());
+
+  // HistogramSynchronizer will Collect histograms from all renderers and it
+  // will call OnHistogramSynchronizationDone (if wait time elapses before it
+  // heard from all renderers, then also it will call
+  // OnHistogramSynchronizationDone).
+
+  // Create a callback_task for OnHistogramSynchronizationDone.
+  Task* callback_task = log_sender_factory_.NewRunnableMethod(
+      &MetricsService::OnHistogramSynchronizationDone);
+
+  // Set up the callback to task to call after we receive histograms from all
+  // renderer processes. Wait time specifies how long to wait before absolutely
+  // calling us back on the task.
+  HistogramSynchronizer::FetchRendererHistogramsAsynchronously(
+      MessageLoop::current(), callback_task,
+      kMaxHistogramGatheringWaitDuration);
+}
+
+void MetricsService::OnHistogramSynchronizationDone() {
   DCHECK(IsSingleThreaded());
 
   // This function should only be called via timer, so timer_pending_
@@ -1035,19 +1076,6 @@ bool MetricsService::TransmissionPermitted() const {
   }
 }
 
-void MetricsService::CollectMemoryDetails() {
-  Task* task = log_sender_factory_.
-      NewRunnableMethod(&MetricsService::TryToStartTransmission);
-  MetricsMemoryDetails* details = new MetricsMemoryDetails(task);
-  details->StartFetch();
-
-  // Collect WebCore cache information to put into a histogram.
-  for (RenderProcessHost::iterator it = RenderProcessHost::begin();
-       it != RenderProcessHost::end(); ++it) {
-    it->second->Send(new ViewMsg_GetCacheResourceStats());
-  }
-}
-
 void MetricsService::PrepareInitialLog() {
   DCHECK(state_ == PLUGIN_LIST_ARRIVED);
   std::vector<WebPluginInfo> plugins;
@@ -1781,21 +1809,9 @@ void MetricsService::RecordCurrentState(PrefService* pref) {
   RecordPluginChanges(pref);
 }
 
-void MetricsService::CollectRendererHistograms() {
-  for (RenderProcessHost::iterator it = RenderProcessHost::begin();
-       it != RenderProcessHost::end(); ++it) {
-    it->second->Send(new ViewMsg_GetRendererHistograms());
-  }
-}
-
 void MetricsService::RecordCurrentHistograms() {
   DCHECK(current_log_);
 
-  CollectRendererHistograms();
-
-  // TODO(raman): Delay the metrics collection activities until we get all the
-  // updates from the renderers, or we time out (1 second?  3 seconds?).
-
   StatisticsRecorder::Histograms histograms;
   StatisticsRecorder::GetHistograms(&histograms);
   for (StatisticsRecorder::Histograms::iterator it = histograms.begin();
diff --git a/chrome/browser/metrics/metrics_service.h b/chrome/browser/metrics/metrics_service.h
index 199e1b1..e78a66a 100644
--- a/chrome/browser/metrics/metrics_service.h
+++ b/chrome/browser/metrics/metrics_service.h
@@ -25,6 +25,7 @@
 
 class BookmarkModel;
 class BookmarkNode;
+class HistogramSynchronizer;
 class PrefService;
 class Profile;
 class TemplateURLModel;
@@ -187,9 +188,15 @@ class MetricsService : public NotificationObserver,
 
   // Start timer for next log transmission.
   void StartLogTransmissionTimer();
-  // Do not call TryToStartTransmission() directly.
+
+  // Internal function to collect process memory information.
+  void LogTransmissionTimerDone();
+
+  // Do not call OnMemoryDetailCollectionDone() or
+  // OnHistogramSynchronizationDone() directly.
   // Use StartLogTransmissionTimer() to schedule a call.
-  void TryToStartTransmission();
+  void OnMemoryDetailCollectionDone();
+  void OnHistogramSynchronizationDone();
 
   // Takes whatever log should be uploaded next (according to the state_)
   // and makes it the pending log.  If pending_log_ is not NULL,
@@ -201,9 +208,6 @@ class MetricsService : public NotificationObserver,
   // TryToStartTransmission.
   bool TransmissionPermitted() const;
 
-  // Internal function to collect process memory information.
-  void CollectMemoryDetails();
-
   // Check to see if there is a log that needs to be, or is being, transmitted.
   bool pending_log() const {
     return pending_log_ || !pending_log_text_.empty();
diff --git a/chrome/browser/renderer_host/resource_message_filter.cc b/chrome/browser/renderer_host/resource_message_filter.cc
index 4749064..8cfaa61 100644
--- a/chrome/browser/renderer_host/resource_message_filter.cc
+++ b/chrome/browser/renderer_host/resource_message_filter.cc
@@ -23,6 +23,7 @@
 #include "chrome/common/app_cache/app_cache_dispatcher_host.h"
 #include "chrome/common/chrome_plugin_lib.h"
 #include "chrome/common/chrome_plugin_util.h"
+#include "chrome/common/histogram_synchronizer.h"
 #include "chrome/common/notification_service.h"
 #include "chrome/common/pref_names.h"
 #include "chrome/common/pref_service.h"
@@ -832,8 +833,9 @@ void ResourceMessageFilter::OnDnsPrefetch(
 }
 
 void ResourceMessageFilter::OnRendererHistograms(
+    int sequence_number,
     const std::vector<std::string>& histograms) {
-  Histogram::DeserializeHistogramList(histograms);
+  HistogramSynchronizer::DeserializeHistogramList(sequence_number, histograms);
 }
 
 #if defined(OS_MACOSX)
diff --git a/chrome/browser/renderer_host/resource_message_filter.h b/chrome/browser/renderer_host/resource_message_filter.h
index bc156c7..9e23079 100644
--- a/chrome/browser/renderer_host/resource_message_filter.h
+++ b/chrome/browser/renderer_host/resource_message_filter.h
@@ -152,7 +152,8 @@ class ResourceMessageFilter : public IPC::ChannelProxy::MessageFilter,
   void OnGetAutoCorrectWord(const std::wstring& word,
                             IPC::Message* reply_msg);
   void OnDnsPrefetch(const std::vector<std::string>& hostnames);
-  void OnRendererHistograms(const std::vector<std::string>& histogram_info);
+  void OnRendererHistograms(int sequence_number,
+                            const std::vector<std::string>& histogram_info);
   void OnReceiveContextMenuMsg(const IPC::Message& msg);
   // Clipboard messages
   void OnClipboardWriteObjects(const Clipboard::ObjectMap& objects);
diff --git a/chrome/chrome.gyp b/chrome/chrome.gyp
index 2765465..5d43dd8 100644
--- a/chrome/chrome.gyp
+++ b/chrome/chrome.gyp
@@ -307,6 +307,8 @@
         'common/gears_api.h',
         'common/gtk_util.cc',
         'common/gtk_util.h',
+        'common/histogram_synchronizer.cc',
+        'common/histogram_synchronizer.h',
         'common/important_file_writer.cc',
         'common/important_file_writer.h',
         'common/ipc_channel.h',
diff --git a/chrome/common/common.vcproj b/chrome/common/common.vcproj
index 9064ac6..dc3ed4d 100644
--- a/chrome/common/common.vcproj
+++ b/chrome/common/common.vcproj
@@ -478,6 +478,14 @@
 			>
 		</File>
 		<File
+			RelativePath=".\histogram_synchronizer.cc"
+			>
+		</File>
+		<File
+			RelativePath=".\histogram_synchronizer.h"
+			>
+		</File>
+		<File
 			RelativePath=".\important_file_writer.cc"
 			>
 		</File>
diff --git a/chrome/common/histogram_synchronizer.cc b/chrome/common/histogram_synchronizer.cc
new file mode 100644
index 0000000..4713084
--- /dev/null
+++ b/chrome/common/histogram_synchronizer.cc
@@ -0,0 +1,260 @@
+// Copyright (c) 2006-2008 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include <string>
+
+#include "base/histogram.h"
+#include "base/logging.h"
+#include "base/string_util.h"
+#include "chrome/browser/browser.h"
+#include "chrome/browser/browser_list.h"
+#include "chrome/browser/browser_process.h"
+#include "chrome/browser/renderer_host/render_process_host.h"
+#include "chrome/common/child_thread.h"
+#include "chrome/common/render_messages.h"
+#include "chrome/common/histogram_synchronizer.h"
+
+using base::Time;
+using base::TimeDelta;
+using base::TimeTicks;
+
+HistogramSynchronizer::HistogramSynchronizer()
+  : lock_(),
+    received_all_renderer_historgrams_(&lock_),
+    callback_task_(NULL),
+    callback_thread_(NULL),
+    io_message_loop_(NULL),
+    next_available_sequence_number_(0),
+    async_sequence_number_(0),
+    async_renderers_pending_(0),
+    async_callback_start_time_(TimeTicks::Now()),
+    synchronous_sequence_number_(0),
+    synchronous_renderers_pending_(0) {
+  DCHECK(histogram_synchronizer_ == NULL);
+  histogram_synchronizer_ = this;
+}
+
+HistogramSynchronizer::~HistogramSynchronizer() {
+  // Clean up.
+  delete callback_task_;
+  callback_task_ = NULL;
+  callback_thread_ = NULL;
+  histogram_synchronizer_ = NULL;
+}
+
+// static
+HistogramSynchronizer* HistogramSynchronizer::CurrentSynchronizer() {
+  DCHECK(histogram_synchronizer_ != NULL);
+  return histogram_synchronizer_;
+}
+
+void HistogramSynchronizer::FetchRendererHistogramsSynchronously(
+    TimeDelta wait_time) {
+  DCHECK(MessageLoop::current()->type() == MessageLoop::TYPE_UI);
+
+  int sequence_number = GetNextAvaibleSequenceNumber(
+      SYNCHRONOUS_HISTOGRAMS, RenderProcessHost::size());
+  for (RenderProcessHost::iterator it = RenderProcessHost::begin();
+       it != RenderProcessHost::end(); ++it) {
+    it->second->Send(new ViewMsg_GetRendererHistograms(sequence_number));
+  }
+
+  TimeTicks start = TimeTicks::Now();
+  TimeTicks end_time = start + wait_time;
+  int unresponsive_renderer_count;
+  {
+    AutoLock auto_lock(lock_);
+    while (synchronous_renderers_pending_ > 0 &&
+           TimeTicks::Now() < end_time) {
+      wait_time = end_time - TimeTicks::Now();
+      received_all_renderer_historgrams_.TimedWait(wait_time);
+    }
+    unresponsive_renderer_count = synchronous_renderers_pending_;
+    synchronous_renderers_pending_ = 0;
+    synchronous_sequence_number_ = 0;
+  }
+  UMA_HISTOGRAM_COUNTS("Histogram.RendersNotRespondingSynchronous",
+                       unresponsive_renderer_count);
+  if (!unresponsive_renderer_count)
+    UMA_HISTOGRAM_TIMES("Histogram.FetchRendererHistogramsSynchronously",
+                        TimeTicks::Now() - start);
+}
+
+// static
+void HistogramSynchronizer::FetchRendererHistogramsAsynchronously(
+    MessageLoop* callback_thread,
+    Task* callback_task,
+    int wait_time) {
+  DCHECK(MessageLoop::current()->type() == MessageLoop::TYPE_UI);
+  DCHECK(callback_thread != NULL);
+  DCHECK(callback_task != NULL);
+
+  HistogramSynchronizer* current_synchronizer =
+      HistogramSynchronizer::CurrentSynchronizer();
+
+  if (current_synchronizer == NULL) {
+    // System teardown is happening.
+    callback_thread->PostTask(FROM_HERE, callback_task);
+    return;
+  }
+
+  // callback_task_ member can only be accessed on IO thread.
+  g_browser_process->io_thread()->message_loop()->PostTask(FROM_HERE,
+      NewRunnableMethod(current_synchronizer,
+          &HistogramSynchronizer::SetCallbackTaskToCallAfterGettingHistograms,
+          callback_thread,
+          callback_task));
+
+  // Tell all renderer processes to send their histograms.
+  int sequence_number =
+      current_synchronizer->GetNextAvaibleSequenceNumber(
+          ASYNC_HISTOGRAMS, RenderProcessHost::size());
+  for (RenderProcessHost::iterator it = RenderProcessHost::begin();
+       it != RenderProcessHost::end(); ++it) {
+    it->second->Send(new ViewMsg_GetRendererHistograms(sequence_number));
+  }
+
+  // Post a task that would be called after waiting for wait_time.
+  g_browser_process->io_thread()->message_loop()->PostDelayedTask(FROM_HERE,
+      NewRunnableMethod(current_synchronizer,
+          &HistogramSynchronizer::ForceHistogramSynchronizationDoneCallback,
+          sequence_number),
+      wait_time);
+}
+
+// static
+void HistogramSynchronizer::DeserializeHistogramList(
+    int sequence_number,
+    const std::vector<std::string>& histograms) {
+  HistogramSynchronizer* current_synchronizer =
+      HistogramSynchronizer::CurrentSynchronizer();
+  if (current_synchronizer == NULL)
+    return;
+
+  DCHECK(current_synchronizer->IsOnIoThread());
+
+  for (std::vector<std::string>::const_iterator it = histograms.begin();
+       it < histograms.end();
+       ++it) {
+    Histogram::DeserializeHistogramInfo(*it);
+  }
+
+  // Record that we have received a histogram from renderer process.
+  current_synchronizer->RecordRendererHistogram(sequence_number);
+}
+
+bool HistogramSynchronizer::RecordRendererHistogram(int sequence_number) {
+  DCHECK(IsOnIoThread());
+
+  if (sequence_number == async_sequence_number_) {
+    if ((async_renderers_pending_ == 0) ||
+        (--async_renderers_pending_ > 0))
+      return false;
+    DCHECK(callback_task_ != NULL);
+    CallCallbackTaskAndResetData();
+    return true;
+  }
+
+  {
+    AutoLock auto_lock(lock_);
+    if (sequence_number != synchronous_sequence_number_) {
+      // No need to do anything if the sequence_number does not match current
+      // synchronous_sequence_number_ or async_sequence_number_.
+      return true;
+    }
+    if ((synchronous_renderers_pending_ == 0) ||
+        (--synchronous_renderers_pending_ > 0))
+      return false;
+    DCHECK_EQ(synchronous_renderers_pending_, 0);
+  }
+
+  // We could call Signal() without holding the lock.
+  received_all_renderer_historgrams_.Signal();
+  return true;
+}
+
+// This method is called on the IO thread.
+void HistogramSynchronizer::SetCallbackTaskToCallAfterGettingHistograms(
+    MessageLoop* callback_thread,
+    Task* callback_task) {
+  DCHECK(IsOnIoThread());
+
+  // Test for the existence of a previous task, and post call to post it if it
+  // exists. We promised to post it after some timeout... and at this point, we
+  // should just force the posting.
+  if (callback_task_ != NULL) {
+    CallCallbackTaskAndResetData();
+  }
+
+  // Assert there was no callback_task_ already.
+  DCHECK(callback_task_ == NULL);
+
+  // Save the thread and the callback_task.
+  DCHECK(callback_thread != NULL);
+  DCHECK(callback_task != NULL);
+  callback_task_ = callback_task;
+  callback_thread_ = callback_thread;
+  async_callback_start_time_ = TimeTicks::Now();
+}
+
+void HistogramSynchronizer::ForceHistogramSynchronizationDoneCallback(
+    int sequence_number) {
+  DCHECK(IsOnIoThread());
+
+  if (sequence_number == async_sequence_number_) {
+     CallCallbackTaskAndResetData();
+  }
+}
+
+// If wait time has elapsed or if we have received all the histograms from all
+// the renderers, call the callback_task if a callback_task exists. This is
+// called on IO Thread.
+void HistogramSynchronizer::CallCallbackTaskAndResetData() {
+  DCHECK(IsOnIoThread());
+
+  // callback_task_ would be set to NULL, if we have heard from all renderers
+  // and we would have called the callback_task already.
+  if (callback_task_ == NULL) {
+    return;
+  }
+
+  UMA_HISTOGRAM_COUNTS("Histogram.RendersNotRespondingAsynchronous",
+                       async_renderers_pending_);
+  if (!async_renderers_pending_)
+    UMA_HISTOGRAM_TIMES("Histogram.FetchRendererHistogramsAsynchronously",
+                        TimeTicks::Now() - async_callback_start_time_);
+
+  DCHECK(callback_thread_ != NULL);
+  DCHECK(callback_task_ != NULL);
+  callback_thread_->PostTask(FROM_HERE, callback_task_);
+  async_renderers_pending_ = 0;
+  async_callback_start_time_ = TimeTicks::Now();
+  callback_task_ = NULL;
+  callback_thread_ = NULL;
+}
+
+int HistogramSynchronizer::GetNextAvaibleSequenceNumber(
+    RendererHistogramRequester requester,
+    size_t renderer_histograms_requested) {
+  AutoLock auto_lock(lock_);
+  ++next_available_sequence_number_;
+  if (requester == ASYNC_HISTOGRAMS) {
+    async_sequence_number_ = next_available_sequence_number_;
+    async_renderers_pending_ = renderer_histograms_requested;
+  } else if (requester == SYNCHRONOUS_HISTOGRAMS) {
+    synchronous_sequence_number_ = next_available_sequence_number_;
+    synchronous_renderers_pending_ = renderer_histograms_requested;
+  }
+  return next_available_sequence_number_;
+}
+
+bool HistogramSynchronizer::IsOnIoThread() {
+  if (io_message_loop_ == NULL) {
+    io_message_loop_ = MessageLoop::current();
+  }
+  return (MessageLoop::current() == io_message_loop_);
+}
+
+// static
+HistogramSynchronizer* HistogramSynchronizer::histogram_synchronizer_ = NULL;
diff --git a/chrome/common/histogram_synchronizer.h b/chrome/common/histogram_synchronizer.h
new file mode 100644
index 0000000..e50002f
--- /dev/null
+++ b/chrome/common/histogram_synchronizer.h
@@ -0,0 +1,148 @@
+// Copyright (c) 2006-2008 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef CHROME_COMMON_HISTOGRAM_SYNCHRONIZER_H_
+#define CHROME_COMMON_HISTOGRAM_SYNCHRONIZER_H_
+
+#include <list>
+#include <map>
+#include <set>
+#include <string>
+#include <vector>
+
+#include "base/basictypes.h"
+#include "base/condition_variable.h"
+#include "base/lock.h"
+#include "base/message_loop.h"
+#include "base/process.h"
+#include "base/ref_counted.h"
+#include "base/scoped_ptr.h"
+#include "base/task.h"
+#include "base/time.h"
+
+class MessageLoop;
+
+class HistogramSynchronizer : public
+    base::RefCountedThreadSafe<HistogramSynchronizer> {
+ public:
+
+  enum RendererHistogramRequester {
+    ASYNC_HISTOGRAMS,
+    SYNCHRONOUS_HISTOGRAMS
+  };
+
+  HistogramSynchronizer();
+
+  ~HistogramSynchronizer();
+
+  // Return pointer to the singleton instance, which is allocated and
+  // deallocated on the main UI thread (during system startup and teardown).
+  static HistogramSynchronizer* CurrentSynchronizer();
+
+  // Contact all renderers, and get them to upload to the browser any/all
+  // changes to histograms.  Return when all changes have been acquired, or when
+  // the wait time expires (whichever is sooner). This method is called on the
+  // main UI thread from about:histograms.
+  void FetchRendererHistogramsSynchronously(base::TimeDelta wait_time);
+
+  // Contact all renderers, and get them to upload to the browser any/all
+  // changes to histograms.  When all changes have been acquired, or when the
+  // wait time expires (whichever is sooner), post the callback_task to the UI
+  // thread. Note the callback_task is posted exactly once. This method is
+  // called on the IO thread from UMA via PostMessage.
+  static void FetchRendererHistogramsAsynchronously(
+      MessageLoop* callback_thread, Task* callback_task, int wait_time);
+
+  // This method is called on the IO thread. Desrializes the histograms and
+  // records that we have received histograms from a renderer process.
+  static void DeserializeHistogramList(
+      int sequence_number, const std::vector<std::string>& histograms);
+
+ private:
+  // Records that we have received the histograms from a renderer for the given
+  // sequence number. If we have received a response from all histograms, either
+  // signal the waiting process or call the callback function. Returns true when
+  // we receive histograms from the last of N renderers that were contacted for
+  // an update. This is called on IO Thread.
+  bool RecordRendererHistogram(int sequence_number);
+
+  void SetCallbackTaskToCallAfterGettingHistograms(
+      MessageLoop* callback_thread, Task* callback_task);
+
+  void ForceHistogramSynchronizationDoneCallback(int sequence_number);
+
+  // Calls the callback task, if there is a callback_task.
+  void CallCallbackTaskAndResetData();
+
+  // Method to get a new sequence number to be sent to renderers from broswer
+  // process.
+  int GetNextAvaibleSequenceNumber(RendererHistogramRequester requster,
+                                   size_t renderer_histograms_requested);
+
+  // For use ONLY in a DCHECK. This method initializes io_message_loop_ in its
+  // first call and then compares io_message_loop_ with MessageLoop::current()
+  // in subsequent calls. This method guarantees we're consistently on the
+  // singular IO thread and we don't need to worry about locks.
+  bool IsOnIoThread();
+
+  // This lock_ protects access to sequence number and
+  // synchronous_renderers_pending_.
+  Lock lock_;
+
+  // This condition variable is used to block caller of the synchronous request
+  // to update histograms, and to signal that thread when updates are completed.
+  ConditionVariable received_all_renderer_historgrams_;
+
+  // When a request is made to asynchronously update the histograms, we store
+  // the task and thread we use to post a completion notification in
+  // callback_task_ and callback_thread_.
+  Task* callback_task_;
+  MessageLoop* callback_thread_;
+
+  // For use ONLY in a DCHECK and is used in IsOnIoThread(). io_message_loop_ is
+  // initialized during the first call to IsOnIoThread(), and then compares
+  // MessageLoop::current() against io_message_loop_ in subsequent calls for
+  // consistency.
+  MessageLoop* io_message_loop_;
+
+  // We don't track the actual renderers that are contacted for an update, only
+  // the count of the number of renderers, and we can sometimes time-out and
+  // give up on a "slow to respond" renderer.  We use a sequence_number to be
+  // sure a response from a renderer is associated with the current round of
+  // requests (and not merely a VERY belated prior response).
+  // next_available_sequence_number_ is the next available number (used to
+  // avoid reuse for a long time).
+  int next_available_sequence_number_;
+
+  // The sequence number used by the most recent asynchronous update request to
+  // contact all renderers.
+  int async_sequence_number_;
+
+  // The number of renderers that have not yet responded to requests (as part of
+  // an asynchronous update).
+  int async_renderers_pending_;
+
+  // The time when we were told to start the fetch histograms asynchronously
+  // from renderers.
+  base::TimeTicks async_callback_start_time_;
+
+  // The sequence number used by the most recent synchronous update request to
+  // contact all renderers.
+  int synchronous_sequence_number_;
+
+  // The number of renderers that have not yet responded to requests (as part of
+  // a synchronous update).
+  int synchronous_renderers_pending_;
+
+  // This singleton instance should be started during the single threaded
+  // portion of main(). It initializes globals to provide support for all future
+  // calls. This object is created on the UI thread, and it is destroyed after
+  // all the other threads have gone away. As a result, it is ok to call it
+  // from the UI thread (for UMA uploads), or for about:histograms.
+  static HistogramSynchronizer* histogram_synchronizer_;
+
+  DISALLOW_EVIL_CONSTRUCTORS(HistogramSynchronizer);
+};
+
+#endif  // CHROME_COMMON_HISTOGRAM_SYNCHRONIZER_H_
diff --git a/chrome/common/render_messages_internal.h b/chrome/common/render_messages_internal.h
index 6b5a5d7..5faeb51 100644
--- a/chrome/common/render_messages_internal.h
+++ b/chrome/common/render_messages_internal.h
@@ -457,7 +457,8 @@ IPC_BEGIN_MESSAGES(View)
   IPC_MESSAGE_CONTROL0(ViewMsg_GetCacheResourceStats)
 
   // Asks the renderer to send back Histograms.
-  IPC_MESSAGE_CONTROL0(ViewMsg_GetRendererHistograms)
+  IPC_MESSAGE_CONTROL1(ViewMsg_GetRendererHistograms,
+                       int /* sequence number of Renderer Histograms. */)
 
   // Notifies the renderer about ui theme changes
   IPC_MESSAGE_ROUTED0(ViewMsg_ThemeChanged)
@@ -1110,7 +1111,9 @@ IPC_BEGIN_MESSAGES(ViewHost)
                       std::wstring /* action */)
 
   // Send back histograms as vector of pickled-histogram strings.
-  IPC_MESSAGE_CONTROL1(ViewHostMsg_RendererHistograms, std::vector<std::string>)
+  IPC_MESSAGE_CONTROL2(ViewHostMsg_RendererHistograms,
+                       int, /* sequence number of Renderer Histograms. */
+                       std::vector<std::string>)
 
   // Request for a DNS prefetch of the names in the array.
   // NameList is typedef'ed std::vector<std::string>
diff --git a/chrome/renderer/render_thread.cc b/chrome/renderer/render_thread.cc
index eae0c55..0412b94 100644
--- a/chrome/renderer/render_thread.cc
+++ b/chrome/renderer/render_thread.cc
@@ -95,8 +95,8 @@ void RenderThread::Resolve(const char* name, size_t length) {
   return dns_master_->Resolve(name, length);
 }
 
-void RenderThread::SendHistograms() {
-  return histogram_snapshots_->SendHistograms();
+void RenderThread::SendHistograms(int sequence_number) {
+  return histogram_snapshots_->SendHistograms(sequence_number);
 }
 
 static WebAppCacheContext* CreateAppCacheContextForRenderer() {
@@ -212,7 +212,7 @@ void RenderThread::OnControlMessageReceived(const IPC::Message& msg) {
     IPC_MESSAGE_HANDLER(ViewMsg_New, OnCreateNewView)
     IPC_MESSAGE_HANDLER(ViewMsg_SetCacheCapacities, OnSetCacheCapacities)
     IPC_MESSAGE_HANDLER(ViewMsg_GetRendererHistograms,
-                          OnGetRendererHistograms)
+                        OnGetRendererHistograms)
     IPC_MESSAGE_HANDLER(ViewMsg_GetCacheResourceStats,
                         OnGetCacheResourceStats)
     IPC_MESSAGE_HANDLER(ViewMsg_UserScripts_UpdatedScripts,
@@ -274,8 +274,8 @@ void RenderThread::OnGetCacheResourceStats() {
   Send(new ViewHostMsg_ResourceTypeStats(stats));
 }
 
-void RenderThread::OnGetRendererHistograms() {
-  SendHistograms();
+void RenderThread::OnGetRendererHistograms(int sequence_number) {
+  SendHistograms(sequence_number);
 }
 
 void RenderThread::InformHostOfCacheStats() {
diff --git a/chrome/renderer/render_thread.h b/chrome/renderer/render_thread.h
index 123c9dc..27312c0 100644
--- a/chrome/renderer/render_thread.h
+++ b/chrome/renderer/render_thread.h
@@ -97,7 +97,7 @@ class RenderThread : public RenderThreadBase,
   void Resolve(const char* name, size_t length);
 
   // Send all the Histogram data to browser.
-  void SendHistograms();
+  void SendHistograms(int sequence_number);
 
   // Invokes InformHostOfCacheStats after a short delay.  Used to move this
   // bookkeeping operation off the critical latency path.
@@ -125,7 +125,7 @@ class RenderThread : public RenderThreadBase,
   void OnGetCacheResourceStats();
 
   // Send all histograms to browser.
-  void OnGetRendererHistograms();
+  void OnGetRendererHistograms(int sequence_number);
 
   void OnExtensionHandleConnect(int channel_id, const std::string& tab_json);
   void OnExtensionHandleMessage(const std::string& message, int channel_id);
diff --git a/chrome/renderer/renderer_histogram_snapshots.cc b/chrome/renderer/renderer_histogram_snapshots.cc
index 6d0507d..025eb73 100644
--- a/chrome/renderer/renderer_histogram_snapshots.cc
+++ b/chrome/renderer/renderer_histogram_snapshots.cc
@@ -22,13 +22,13 @@ RendererHistogramSnapshots::RendererHistogramSnapshots()
 }
 
 // Send data quickly!
-void RendererHistogramSnapshots::SendHistograms() {
+void RendererHistogramSnapshots::SendHistograms(int sequence_number) {
   RenderThread::current()->message_loop()->PostTask(FROM_HERE,
       renderer_histogram_snapshots_factory_.NewRunnableMethod(
-          &RendererHistogramSnapshots::UploadAllHistrograms));
+          &RendererHistogramSnapshots::UploadAllHistrograms, sequence_number));
 }
 
-void RendererHistogramSnapshots::UploadAllHistrograms() {
+void RendererHistogramSnapshots::UploadAllHistrograms(int sequence_number) {
   StatisticsRecorder::Histograms histograms;
   StatisticsRecorder::GetHistograms(&histograms);
 
@@ -39,11 +39,11 @@ void RendererHistogramSnapshots::UploadAllHistrograms() {
        it++) {
     UploadHistrogram(**it, &pickled_histograms);
   }
-  // Send the handle over synchronous IPC.
-  if (pickled_histograms.size() > 0) {
-    RenderThread::current()->Send(
-        new ViewHostMsg_RendererHistograms(pickled_histograms));
-  }
+  // Send the sequence number and list of pickled histograms over synchronous
+  // IPC.
+  RenderThread::current()->Send(
+      new ViewHostMsg_RendererHistograms(
+          sequence_number, pickled_histograms));
 }
 
 // Extract snapshot data and then send it off the the Browser process
diff --git a/chrome/renderer/renderer_histogram_snapshots.h b/chrome/renderer/renderer_histogram_snapshots.h
index 5c9f93a..0a1238c 100644
--- a/chrome/renderer/renderer_histogram_snapshots.h
+++ b/chrome/renderer/renderer_histogram_snapshots.h
@@ -24,7 +24,7 @@ class RendererHistogramSnapshots {
   ~RendererHistogramSnapshots() {}
 
   // Send the histogram data.
-  void SendHistograms();
+  void SendHistograms(int sequence_number);
 
   // Maintain a map of histogram names to the sample stats we've sent.
   typedef std::map<std::string, Histogram::SampleSet> LoggedSampleMap;
@@ -33,7 +33,7 @@ class RendererHistogramSnapshots {
  private:
   // Extract snapshot data and then send it off the the Browser process.
   // Send only a delta to what we have already sent.
-  void UploadAllHistrograms();
+  void UploadAllHistrograms(int sequence_number);
   void UploadHistrogram(const Histogram& histogram,
                         HistogramPickledList* histograms);
   void UploadHistogramDelta(const Histogram& histogram,