When ThreadWatcher detects that a thread is not responding,

record how many other threads are responding and how many
are not responding. This is to get more data before we crash
the browser when we detect threads are not responding (which
could help us to detect the dead locks or hung threads.

BUG=80751
Test=thread watcher unit tests
R=jar
Review URL: http://codereview.chromium.org/6905084

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@83432 0039d316-1c4b-4281-b951-d872f2087c98

2 files changed, 48 insertions, 17 deletions

diff --git a/chrome/browser/metrics/thread_watcher.cc b/chrome/browser/metrics/thread_watcher.cc
index fcc518a..7d7732e 100644
--- a/chrome/browser/metrics/thread_watcher.cc
+++ b/chrome/browser/metrics/thread_watcher.cc
@@ -207,6 +207,18 @@ void ThreadWatcher::Initialize() {
       base::TimeDelta::FromMilliseconds(1),
       base::TimeDelta::FromSeconds(100), 50,
       base::Histogram::kUmaTargetedHistogramFlag);
+
+  const std::string responsive_count_histogram_name =
+      "ThreadWatcher.ResponsiveThreads." + thread_name_;
+  responsive_count_histogram_ = base::LinearHistogram::FactoryGet(
+      responsive_count_histogram_name, 1, 10, 11,
+      base::Histogram::kUmaTargetedHistogramFlag);
+
+  const std::string unresponsive_count_histogram_name =
+      "ThreadWatcher.UnresponsiveThreads." + thread_name_;
+  unresponsive_count_histogram_ = base::LinearHistogram::FactoryGet(
+      unresponsive_count_histogram_name, 1, 10, 11,
+      base::Histogram::kUmaTargetedHistogramFlag);
 }
 
 // static
@@ -225,12 +237,20 @@ void ThreadWatcher::GotGoodResponse() {
 void ThreadWatcher::GotNoResponse() {
   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
   ++unresponsive_count_;
-  // If watched thread is the only unresponsive thread and all other threads are
-  // responding then record total unresponsive_time since last pong message.
-  if (ThreadWatcherList::GetNumberOfUnresponsiveThreads() == 1) {
-    base::TimeDelta unresponse_time = base::TimeTicks::Now() - pong_time_;
-    unresponsive_time_histogram_->AddTime(unresponse_time);
-  }
+  // Record total unresponsive_time since last pong message.
+  base::TimeDelta unresponse_time = base::TimeTicks::Now() - pong_time_;
+  unresponsive_time_histogram_->AddTime(unresponse_time);
+
+  int no_of_responding_threads = 0;
+  int no_of_unresponding_threads = 0;
+  ThreadWatcherList::GetStatusOfThreads(&no_of_responding_threads,
+                                        &no_of_unresponding_threads);
+
+  // Record how many watched threads are responding.
+  responsive_count_histogram_->Add(no_of_responding_threads);
+
+  // Record how many watched threads are not responding.
+  unresponsive_count_histogram_->Add(no_of_unresponding_threads);
 }
 
 // ThreadWatcherList methods and members.
@@ -338,20 +358,23 @@ void ThreadWatcherList::RemoveNotifications() {
 }
 
 // static
-int ThreadWatcherList::GetNumberOfUnresponsiveThreads() {
+void ThreadWatcherList::GetStatusOfThreads(int* no_of_responding_threads,
+                                           int* no_of_unresponding_threads) {
   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
-  int no_of_unresponding_threads = 0;
+  *no_of_responding_threads = 0;
+  *no_of_unresponding_threads = 0;
   if (!global_)
-    return no_of_unresponding_threads;
+    return;
 
   base::AutoLock auto_lock(global_->lock_);
   for (RegistrationList::iterator it = global_->registered_.begin();
        global_->registered_.end() != it;
        ++it) {
     if (it->second->unresponsive_count_ > 0)
-      ++no_of_unresponding_threads;
+      ++(*no_of_unresponding_threads);
+    else
+      ++(*no_of_responding_threads);
   }
-  return no_of_unresponding_threads;
 }
 
 void ThreadWatcherList::DeleteAll() {
diff --git a/chrome/browser/metrics/thread_watcher.h b/chrome/browser/metrics/thread_watcher.h
index 5402c5f..5387493 100644
--- a/chrome/browser/metrics/thread_watcher.h
+++ b/chrome/browser/metrics/thread_watcher.h
@@ -200,11 +200,18 @@ class ThreadWatcher {
   base::Histogram* response_time_histogram_;
 
   // Histogram that keeps track of unresponsive time since the last pong message
-  // when we got no response (GotNoResponse) from the watched thread. We record
-  // this when the watched thread is the only unresponsive thread and all other
-  // threads are responsive.
+  // when we got no response (GotNoResponse) from the watched thread.
   base::Histogram* unresponsive_time_histogram_;
 
+  // Histogram that keeps track of how many threads are responding when we got
+  // no response (GotNoResponse) from the watched thread.
+  base::Histogram* responsive_count_histogram_;
+
+  // Histogram that keeps track of how many threads are not responding when we
+  // got no response (GotNoResponse) from the watched thread. Count includes the
+  // thread that got no response.
+  base::Histogram* unresponsive_count_histogram_;
+
   // This counter tracks the unresponsiveness of watched thread. If this value
   // is zero then watched thread has responded with a pong message. This is
   // incremented by 1 when we got no response (GotNoResponse) from the watched
@@ -254,9 +261,10 @@ class ThreadWatcherList : public NotificationObserver {
   // This method is accessible on UI thread.
   static void RemoveNotifications();
 
-  // This method returns number of watched threads that haven't responded with a
-  // pong message (number of threads with unresponsive_count_ greater than 0).
-  static int GetNumberOfUnresponsiveThreads();
+  // This method returns number of watched threads that have responded and
+  // threads that have not responded with a pong message.
+  static void GetStatusOfThreads(int* no_of_responding_threads,
+                                 int* no_of_unresponding_threads);
 
  private:
   // Allow tests to access our innards for testing purposes.