Automatically adapt to faster/slower uploads of renderer histograms

This replaces the current time based approach (chrome is given N seconds
to upload all renederer histograms) with an asynch callback approach
that waits until all renderers have responded (with their updates). It
uses a fall-back timer to ensure that a hung renderer won't delay things
forever as well.

This causes faster (and complete) updates in about:histograms as well
as generally assuring complete updates during UMA gatherings.

This code was contributed by Raman Tenneti in CL 42496
http://codereview.chromium.org/42496

bug=12850

r=raman
Review URL: http://codereview.chromium.org/113473

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@17123 0039d316-1c4b-4281-b951-d872f2087c98

4 files changed, 421 insertions, 2 deletions

diff --git a/chrome/common/common.vcproj b/chrome/common/common.vcproj
index 9064ac6..dc3ed4d 100644
--- a/chrome/common/common.vcproj
+++ b/chrome/common/common.vcproj
@@ -478,6 +478,14 @@
 			>
 		</File>
 		<File
+			RelativePath=".\histogram_synchronizer.cc"
+			>
+		</File>
+		<File
+			RelativePath=".\histogram_synchronizer.h"
+			>
+		</File>
+		<File
 			RelativePath=".\important_file_writer.cc"
 			>
 		</File>
diff --git a/chrome/common/histogram_synchronizer.cc b/chrome/common/histogram_synchronizer.cc
new file mode 100644
index 0000000..4713084
--- /dev/null
+++ b/chrome/common/histogram_synchronizer.cc
@@ -0,0 +1,260 @@
+// Copyright (c) 2006-2008 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include <string>
+
+#include "base/histogram.h"
+#include "base/logging.h"
+#include "base/string_util.h"
+#include "chrome/browser/browser.h"
+#include "chrome/browser/browser_list.h"
+#include "chrome/browser/browser_process.h"
+#include "chrome/browser/renderer_host/render_process_host.h"
+#include "chrome/common/child_thread.h"
+#include "chrome/common/render_messages.h"
+#include "chrome/common/histogram_synchronizer.h"
+
+using base::Time;
+using base::TimeDelta;
+using base::TimeTicks;
+
+HistogramSynchronizer::HistogramSynchronizer()
+  : lock_(),
+    received_all_renderer_historgrams_(&lock_),
+    callback_task_(NULL),
+    callback_thread_(NULL),
+    io_message_loop_(NULL),
+    next_available_sequence_number_(0),
+    async_sequence_number_(0),
+    async_renderers_pending_(0),
+    async_callback_start_time_(TimeTicks::Now()),
+    synchronous_sequence_number_(0),
+    synchronous_renderers_pending_(0) {
+  DCHECK(histogram_synchronizer_ == NULL);
+  histogram_synchronizer_ = this;
+}
+
+HistogramSynchronizer::~HistogramSynchronizer() {
+  // Clean up.
+  delete callback_task_;
+  callback_task_ = NULL;
+  callback_thread_ = NULL;
+  histogram_synchronizer_ = NULL;
+}
+
+// static
+HistogramSynchronizer* HistogramSynchronizer::CurrentSynchronizer() {
+  DCHECK(histogram_synchronizer_ != NULL);
+  return histogram_synchronizer_;
+}
+
+void HistogramSynchronizer::FetchRendererHistogramsSynchronously(
+    TimeDelta wait_time) {
+  DCHECK(MessageLoop::current()->type() == MessageLoop::TYPE_UI);
+
+  int sequence_number = GetNextAvaibleSequenceNumber(
+      SYNCHRONOUS_HISTOGRAMS, RenderProcessHost::size());
+  for (RenderProcessHost::iterator it = RenderProcessHost::begin();
+       it != RenderProcessHost::end(); ++it) {
+    it->second->Send(new ViewMsg_GetRendererHistograms(sequence_number));
+  }
+
+  TimeTicks start = TimeTicks::Now();
+  TimeTicks end_time = start + wait_time;
+  int unresponsive_renderer_count;
+  {
+    AutoLock auto_lock(lock_);
+    while (synchronous_renderers_pending_ > 0 &&
+           TimeTicks::Now() < end_time) {
+      wait_time = end_time - TimeTicks::Now();
+      received_all_renderer_historgrams_.TimedWait(wait_time);
+    }
+    unresponsive_renderer_count = synchronous_renderers_pending_;
+    synchronous_renderers_pending_ = 0;
+    synchronous_sequence_number_ = 0;
+  }
+  UMA_HISTOGRAM_COUNTS("Histogram.RendersNotRespondingSynchronous",
+                       unresponsive_renderer_count);
+  if (!unresponsive_renderer_count)
+    UMA_HISTOGRAM_TIMES("Histogram.FetchRendererHistogramsSynchronously",
+                        TimeTicks::Now() - start);
+}
+
+// static
+void HistogramSynchronizer::FetchRendererHistogramsAsynchronously(
+    MessageLoop* callback_thread,
+    Task* callback_task,
+    int wait_time) {
+  DCHECK(MessageLoop::current()->type() == MessageLoop::TYPE_UI);
+  DCHECK(callback_thread != NULL);
+  DCHECK(callback_task != NULL);
+
+  HistogramSynchronizer* current_synchronizer =
+      HistogramSynchronizer::CurrentSynchronizer();
+
+  if (current_synchronizer == NULL) {
+    // System teardown is happening.
+    callback_thread->PostTask(FROM_HERE, callback_task);
+    return;
+  }
+
+  // callback_task_ member can only be accessed on IO thread.
+  g_browser_process->io_thread()->message_loop()->PostTask(FROM_HERE,
+      NewRunnableMethod(current_synchronizer,
+          &HistogramSynchronizer::SetCallbackTaskToCallAfterGettingHistograms,
+          callback_thread,
+          callback_task));
+
+  // Tell all renderer processes to send their histograms.
+  int sequence_number =
+      current_synchronizer->GetNextAvaibleSequenceNumber(
+          ASYNC_HISTOGRAMS, RenderProcessHost::size());
+  for (RenderProcessHost::iterator it = RenderProcessHost::begin();
+       it != RenderProcessHost::end(); ++it) {
+    it->second->Send(new ViewMsg_GetRendererHistograms(sequence_number));
+  }
+
+  // Post a task that would be called after waiting for wait_time.
+  g_browser_process->io_thread()->message_loop()->PostDelayedTask(FROM_HERE,
+      NewRunnableMethod(current_synchronizer,
+          &HistogramSynchronizer::ForceHistogramSynchronizationDoneCallback,
+          sequence_number),
+      wait_time);
+}
+
+// static
+void HistogramSynchronizer::DeserializeHistogramList(
+    int sequence_number,
+    const std::vector<std::string>& histograms) {
+  HistogramSynchronizer* current_synchronizer =
+      HistogramSynchronizer::CurrentSynchronizer();
+  if (current_synchronizer == NULL)
+    return;
+
+  DCHECK(current_synchronizer->IsOnIoThread());
+
+  for (std::vector<std::string>::const_iterator it = histograms.begin();
+       it < histograms.end();
+       ++it) {
+    Histogram::DeserializeHistogramInfo(*it);
+  }
+
+  // Record that we have received a histogram from renderer process.
+  current_synchronizer->RecordRendererHistogram(sequence_number);
+}
+
+bool HistogramSynchronizer::RecordRendererHistogram(int sequence_number) {
+  DCHECK(IsOnIoThread());
+
+  if (sequence_number == async_sequence_number_) {
+    if ((async_renderers_pending_ == 0) ||
+        (--async_renderers_pending_ > 0))
+      return false;
+    DCHECK(callback_task_ != NULL);
+    CallCallbackTaskAndResetData();
+    return true;
+  }
+
+  {
+    AutoLock auto_lock(lock_);
+    if (sequence_number != synchronous_sequence_number_) {
+      // No need to do anything if the sequence_number does not match current
+      // synchronous_sequence_number_ or async_sequence_number_.
+      return true;
+    }
+    if ((synchronous_renderers_pending_ == 0) ||
+        (--synchronous_renderers_pending_ > 0))
+      return false;
+    DCHECK_EQ(synchronous_renderers_pending_, 0);
+  }
+
+  // We could call Signal() without holding the lock.
+  received_all_renderer_historgrams_.Signal();
+  return true;
+}
+
+// This method is called on the IO thread.
+void HistogramSynchronizer::SetCallbackTaskToCallAfterGettingHistograms(
+    MessageLoop* callback_thread,
+    Task* callback_task) {
+  DCHECK(IsOnIoThread());
+
+  // Test for the existence of a previous task, and post call to post it if it
+  // exists. We promised to post it after some timeout... and at this point, we
+  // should just force the posting.
+  if (callback_task_ != NULL) {
+    CallCallbackTaskAndResetData();
+  }
+
+  // Assert there was no callback_task_ already.
+  DCHECK(callback_task_ == NULL);
+
+  // Save the thread and the callback_task.
+  DCHECK(callback_thread != NULL);
+  DCHECK(callback_task != NULL);
+  callback_task_ = callback_task;
+  callback_thread_ = callback_thread;
+  async_callback_start_time_ = TimeTicks::Now();
+}
+
+void HistogramSynchronizer::ForceHistogramSynchronizationDoneCallback(
+    int sequence_number) {
+  DCHECK(IsOnIoThread());
+
+  if (sequence_number == async_sequence_number_) {
+     CallCallbackTaskAndResetData();
+  }
+}
+
+// If wait time has elapsed or if we have received all the histograms from all
+// the renderers, call the callback_task if a callback_task exists. This is
+// called on IO Thread.
+void HistogramSynchronizer::CallCallbackTaskAndResetData() {
+  DCHECK(IsOnIoThread());
+
+  // callback_task_ would be set to NULL, if we have heard from all renderers
+  // and we would have called the callback_task already.
+  if (callback_task_ == NULL) {
+    return;
+  }
+
+  UMA_HISTOGRAM_COUNTS("Histogram.RendersNotRespondingAsynchronous",
+                       async_renderers_pending_);
+  if (!async_renderers_pending_)
+    UMA_HISTOGRAM_TIMES("Histogram.FetchRendererHistogramsAsynchronously",
+                        TimeTicks::Now() - async_callback_start_time_);
+
+  DCHECK(callback_thread_ != NULL);
+  DCHECK(callback_task_ != NULL);
+  callback_thread_->PostTask(FROM_HERE, callback_task_);
+  async_renderers_pending_ = 0;
+  async_callback_start_time_ = TimeTicks::Now();
+  callback_task_ = NULL;
+  callback_thread_ = NULL;
+}
+
+int HistogramSynchronizer::GetNextAvaibleSequenceNumber(
+    RendererHistogramRequester requester,
+    size_t renderer_histograms_requested) {
+  AutoLock auto_lock(lock_);
+  ++next_available_sequence_number_;
+  if (requester == ASYNC_HISTOGRAMS) {
+    async_sequence_number_ = next_available_sequence_number_;
+    async_renderers_pending_ = renderer_histograms_requested;
+  } else if (requester == SYNCHRONOUS_HISTOGRAMS) {
+    synchronous_sequence_number_ = next_available_sequence_number_;
+    synchronous_renderers_pending_ = renderer_histograms_requested;
+  }
+  return next_available_sequence_number_;
+}
+
+bool HistogramSynchronizer::IsOnIoThread() {
+  if (io_message_loop_ == NULL) {
+    io_message_loop_ = MessageLoop::current();
+  }
+  return (MessageLoop::current() == io_message_loop_);
+}
+
+// static
+HistogramSynchronizer* HistogramSynchronizer::histogram_synchronizer_ = NULL;
diff --git a/chrome/common/histogram_synchronizer.h b/chrome/common/histogram_synchronizer.h
new file mode 100644
index 0000000..e50002f
--- /dev/null
+++ b/chrome/common/histogram_synchronizer.h
@@ -0,0 +1,148 @@
+// Copyright (c) 2006-2008 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef CHROME_COMMON_HISTOGRAM_SYNCHRONIZER_H_
+#define CHROME_COMMON_HISTOGRAM_SYNCHRONIZER_H_
+
+#include <list>
+#include <map>
+#include <set>
+#include <string>
+#include <vector>
+
+#include "base/basictypes.h"
+#include "base/condition_variable.h"
+#include "base/lock.h"
+#include "base/message_loop.h"
+#include "base/process.h"
+#include "base/ref_counted.h"
+#include "base/scoped_ptr.h"
+#include "base/task.h"
+#include "base/time.h"
+
+class MessageLoop;
+
+class HistogramSynchronizer : public
+    base::RefCountedThreadSafe<HistogramSynchronizer> {
+ public:
+
+  enum RendererHistogramRequester {
+    ASYNC_HISTOGRAMS,
+    SYNCHRONOUS_HISTOGRAMS
+  };
+
+  HistogramSynchronizer();
+
+  ~HistogramSynchronizer();
+
+  // Return pointer to the singleton instance, which is allocated and
+  // deallocated on the main UI thread (during system startup and teardown).
+  static HistogramSynchronizer* CurrentSynchronizer();
+
+  // Contact all renderers, and get them to upload to the browser any/all
+  // changes to histograms.  Return when all changes have been acquired, or when
+  // the wait time expires (whichever is sooner). This method is called on the
+  // main UI thread from about:histograms.
+  void FetchRendererHistogramsSynchronously(base::TimeDelta wait_time);
+
+  // Contact all renderers, and get them to upload to the browser any/all
+  // changes to histograms.  When all changes have been acquired, or when the
+  // wait time expires (whichever is sooner), post the callback_task to the UI
+  // thread. Note the callback_task is posted exactly once. This method is
+  // called on the IO thread from UMA via PostMessage.
+  static void FetchRendererHistogramsAsynchronously(
+      MessageLoop* callback_thread, Task* callback_task, int wait_time);
+
+  // This method is called on the IO thread. Desrializes the histograms and
+  // records that we have received histograms from a renderer process.
+  static void DeserializeHistogramList(
+      int sequence_number, const std::vector<std::string>& histograms);
+
+ private:
+  // Records that we have received the histograms from a renderer for the given
+  // sequence number. If we have received a response from all histograms, either
+  // signal the waiting process or call the callback function. Returns true when
+  // we receive histograms from the last of N renderers that were contacted for
+  // an update. This is called on IO Thread.
+  bool RecordRendererHistogram(int sequence_number);
+
+  void SetCallbackTaskToCallAfterGettingHistograms(
+      MessageLoop* callback_thread, Task* callback_task);
+
+  void ForceHistogramSynchronizationDoneCallback(int sequence_number);
+
+  // Calls the callback task, if there is a callback_task.
+  void CallCallbackTaskAndResetData();
+
+  // Method to get a new sequence number to be sent to renderers from broswer
+  // process.
+  int GetNextAvaibleSequenceNumber(RendererHistogramRequester requster,
+                                   size_t renderer_histograms_requested);
+
+  // For use ONLY in a DCHECK. This method initializes io_message_loop_ in its
+  // first call and then compares io_message_loop_ with MessageLoop::current()
+  // in subsequent calls. This method guarantees we're consistently on the
+  // singular IO thread and we don't need to worry about locks.
+  bool IsOnIoThread();
+
+  // This lock_ protects access to sequence number and
+  // synchronous_renderers_pending_.
+  Lock lock_;
+
+  // This condition variable is used to block caller of the synchronous request
+  // to update histograms, and to signal that thread when updates are completed.
+  ConditionVariable received_all_renderer_historgrams_;
+
+  // When a request is made to asynchronously update the histograms, we store
+  // the task and thread we use to post a completion notification in
+  // callback_task_ and callback_thread_.
+  Task* callback_task_;
+  MessageLoop* callback_thread_;
+
+  // For use ONLY in a DCHECK and is used in IsOnIoThread(). io_message_loop_ is
+  // initialized during the first call to IsOnIoThread(), and then compares
+  // MessageLoop::current() against io_message_loop_ in subsequent calls for
+  // consistency.
+  MessageLoop* io_message_loop_;
+
+  // We don't track the actual renderers that are contacted for an update, only
+  // the count of the number of renderers, and we can sometimes time-out and
+  // give up on a "slow to respond" renderer.  We use a sequence_number to be
+  // sure a response from a renderer is associated with the current round of
+  // requests (and not merely a VERY belated prior response).
+  // next_available_sequence_number_ is the next available number (used to
+  // avoid reuse for a long time).
+  int next_available_sequence_number_;
+
+  // The sequence number used by the most recent asynchronous update request to
+  // contact all renderers.
+  int async_sequence_number_;
+
+  // The number of renderers that have not yet responded to requests (as part of
+  // an asynchronous update).
+  int async_renderers_pending_;
+
+  // The time when we were told to start the fetch histograms asynchronously
+  // from renderers.
+  base::TimeTicks async_callback_start_time_;
+
+  // The sequence number used by the most recent synchronous update request to
+  // contact all renderers.
+  int synchronous_sequence_number_;
+
+  // The number of renderers that have not yet responded to requests (as part of
+  // a synchronous update).
+  int synchronous_renderers_pending_;
+
+  // This singleton instance should be started during the single threaded
+  // portion of main(). It initializes globals to provide support for all future
+  // calls. This object is created on the UI thread, and it is destroyed after
+  // all the other threads have gone away. As a result, it is ok to call it
+  // from the UI thread (for UMA uploads), or for about:histograms.
+  static HistogramSynchronizer* histogram_synchronizer_;
+
+  DISALLOW_EVIL_CONSTRUCTORS(HistogramSynchronizer);
+};
+
+#endif  // CHROME_COMMON_HISTOGRAM_SYNCHRONIZER_H_
diff --git a/chrome/common/render_messages_internal.h b/chrome/common/render_messages_internal.h
index 6b5a5d7..5faeb51 100644
--- a/chrome/common/render_messages_internal.h
+++ b/chrome/common/render_messages_internal.h
@@ -457,7 +457,8 @@ IPC_BEGIN_MESSAGES(View)
   IPC_MESSAGE_CONTROL0(ViewMsg_GetCacheResourceStats)
 
   // Asks the renderer to send back Histograms.
-  IPC_MESSAGE_CONTROL0(ViewMsg_GetRendererHistograms)
+  IPC_MESSAGE_CONTROL1(ViewMsg_GetRendererHistograms,
+                       int /* sequence number of Renderer Histograms. */)
 
   // Notifies the renderer about ui theme changes
   IPC_MESSAGE_ROUTED0(ViewMsg_ThemeChanged)
@@ -1110,7 +1111,9 @@ IPC_BEGIN_MESSAGES(ViewHost)
                       std::wstring /* action */)
 
   // Send back histograms as vector of pickled-histogram strings.
-  IPC_MESSAGE_CONTROL1(ViewHostMsg_RendererHistograms, std::vector<std::string>)
+  IPC_MESSAGE_CONTROL2(ViewHostMsg_RendererHistograms,
+                       int, /* sequence number of Renderer Histograms. */
+                       std::vector<std::string>)
 
   // Request for a DNS prefetch of the names in the array.
   // NameList is typedef'ed std::vector<std::string>