Construct a field trial to see if HIGH or MEDIUM memory model "works better"

Includes definition of a FieldTrial class to support this.
I have thoughts in my head about how this will eventually
extend to be controllable via UMA (as well as being able
to run tests defined at compile time, as was done in this
example.


r=mbelshe, mmentovai
Review URL: http://codereview.chromium.org/7638

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@3604 0039d316-1c4b-4281-b951-d872f2087c98

12 files changed, 323 insertions, 14 deletions

diff --git a/base/SConscript b/base/SConscript
index ef462d9..82d652e 100644
--- a/base/SConscript
+++ b/base/SConscript
@@ -35,6 +35,7 @@ input_files = [
     'bzip2_error_handler.cc',
     'command_line.cc',
     'debug_util.cc',
+    'field_trial.cc',
     'file_path.cc',
     'file_util.cc',
     'histogram.cc',
@@ -257,6 +258,7 @@ test_files = [
     'clipboard_unittest.cc',
     'command_line_unittest.cc',
     'condition_variable_unittest.cc',
+    'field_trial_unittest.cc',
     'file_path_unittest.cc',
     'file_util_unittest.cc',
     'hmac_unittest.cc',
diff --git a/base/build/base.vcproj b/base/build/base.vcproj
index 76a08e9..0350c33 100644
--- a/base/build/base.vcproj
+++ b/base/build/base.vcproj
@@ -286,6 +286,14 @@
 			>
 		</File>
 		<File
+			RelativePath="..\field_trial.cc"
+			>
+		</File>
+		<File
+			RelativePath="..\field_trial.h"
+			>
+		</File>
+		<File
 			RelativePath="..\file_path.cc"
 			>
 		</File>
@@ -538,10 +546,6 @@
 			>
 		</File>
 		<File
-			RelativePath="..\process_win.cc"
-			>
-		</File>
-		<File
 			RelativePath="..\process.h"
 			>
 		</File>
@@ -554,6 +558,10 @@
 			>
 		</File>
 		<File
+			RelativePath="..\process_win.cc"
+			>
+		</File>
+		<File
 			RelativePath="..\third_party\nspr\prtime.cc"
 			>
 		</File>
diff --git a/base/build/base_unittests.vcproj b/base/build/base_unittests.vcproj
index 7df06c9..cf91226 100644
--- a/base/build/base_unittests.vcproj
+++ b/base/build/base_unittests.vcproj
@@ -188,6 +188,10 @@
 				>
 			</File>
 			<File
+				RelativePath="..\field_trial_unittest.cc"
+				>
+			</File>
+			<File
 				RelativePath="..\file_path_unittest.cc"
 				>
 			</File>
diff --git a/base/field_trial.cc b/base/field_trial.cc
new file mode 100644
index 0000000..103e539
--- /dev/null
+++ b/base/field_trial.cc
@@ -0,0 +1,62 @@
+// Copyright (c) 2006-2008 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+
+#include "base/field_trial.h"
+#include "base/logging.h"
+#include "base/rand_util.h"
+
+//------------------------------------------------------------------------------
+// FieldTrialList methods and members.
+
+// static
+FieldTrialList* FieldTrialList::global_ = NULL;
+
+// static
+int FieldTrialList::constructor_count_ = 0;
+
+FieldTrialList::FieldTrialList()
+  : application_start_time_(Time::Now()) {
+  DCHECK(!constructor_count_);
+  ++constructor_count_;
+  global_ = this;
+}
+
+FieldTrialList::~FieldTrialList() {
+  while (!registered_.empty()) {
+    RegistrationList::iterator it = registered_.begin();
+    it->second->Release();
+    registered_.erase(it->first);
+  }
+  DCHECK(this == global_);
+  global_ = NULL;
+}
+
+// static
+void FieldTrialList::Register(FieldTrial* trial) {
+  DCHECK(global_->CalledOnValidThread());
+  DCHECK(!Find(trial->name()));
+  trial->AddRef();
+  global_->registered_[trial->name()] = trial;
+}
+
+// static
+FieldTrial* FieldTrialList::Find(const std::wstring& name) {
+  DCHECK(global_->CalledOnValidThread());
+  RegistrationList::iterator it = global_->registered_.find(name);
+  if (global_->registered_.end() == it)
+    return NULL;
+  return it->second;
+}
+
+//------------------------------------------------------------------------------
+// FieldTrial methods and members.
+
+FieldTrial::FieldTrial(const std::wstring& name, double probability)
+  : name_(name) {
+  double rand = base::RandDouble();
+  DCHECK(rand >= 0.0  && rand < 1.0);
+  boolean_value_ = rand < probability;
+  FieldTrialList::Register(this);
+}
diff --git a/base/field_trial.h b/base/field_trial.h
new file mode 100644
index 0000000..e70ab1a
--- /dev/null
+++ b/base/field_trial.h
@@ -0,0 +1,92 @@
+// Copyright (c) 2006-2008 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// FieldTrial is a class for handling details of statistical experiments
+// performed by actual users in the field (i.e., in a shipped or beta product).
+// All code is called exclusively on the UI thread currently.
+//
+// The simplest example is a test to see whether one of two options produces
+// "better" results across our user population.  In that scenario, UMA data
+// is uploaded to show the test results, and this class manages the state of
+// each such test (state == which option was pseudo-randomly selected).
+// States are typically generated randomly, either based on a one time
+// randomization (reused during each run of the program), or by a startup
+// randomization (keeping that tests state constant across a run), or by
+// continuous randomization across a run.
+// Only startup randomization is implemented (thus far).
+
+#ifndef BASE_FIELD_TRIAL_H_
+#define BASE_FIELD_TRIAL_H_
+
+#include <map>
+#include <string>
+
+#include "base/non_thread_safe.h"
+#include "base/ref_counted.h"
+#include "base/time.h"
+
+class FieldTrial : public base::RefCounted<FieldTrial> {
+ public:
+  // Constructor for a 2-state (boolean) trial.
+  // The name is used to register the instance with the FieldTrialList class,
+  // and can be used to find the trial (only one trial can be present for each
+  // name) using the Find() method.
+  // The probability is a number in the range [0, 1], and is the likliehood that
+  // the assigned boolean value will be true.
+  FieldTrial(const std::wstring& name, double probability);
+
+  // Return the selected boolean value.
+  bool boolean_value() const { return boolean_value_; }
+  std::wstring name() const { return name_; }
+
+ private:
+  const std::wstring name_;
+  bool boolean_value_;
+
+  DISALLOW_COPY_AND_ASSIGN(FieldTrial);
+};
+
+// Class with a list of all active field trials.  A trial is active if it has
+// been registered, which includes evaluating its state based on its probaility.
+// Only one instance of this class exists.
+class FieldTrialList : NonThreadSafe {
+ public:
+  // This singleton holds the global list of registered FieldTrials.
+  FieldTrialList();
+  // Destructor Release()'s references to all registered FieldTrial instances.
+  ~FieldTrialList();
+
+  // Register() stores a pointer to the given trial in a global map.
+  // This method also AddRef's the indicated trial.
+  static void Register(FieldTrial* trial);
+
+  // The Find() method can be used to test to see if a named Trial was already
+  // registered, or to retrieve a pointer to it from the global map.
+  static FieldTrial* Find(const std::wstring& name);
+
+  // The time of construction of the global map is recorded in a static variable
+  // and is commonly used by experiments to identify the time since the start
+  // of the application.  In some experiments it may be useful to discount
+  // data that is gathered before the application has reach sufficient
+  // stability (example: most DLL have loaded, etc.)
+  static Time application_start_time() {
+    return global_->application_start_time_;
+  }
+
+ private:
+  typedef std::map<std::wstring, FieldTrial*> RegistrationList;
+
+  friend class FieldTrialTest;
+  static void ResetConstructorCountForTestingOnly() { constructor_count_ = 0; }
+
+  static FieldTrialList* global_;  // The singleton of this class.
+  static int constructor_count_;   // Prevent having more than one.
+
+  Time application_start_time_;
+  RegistrationList registered_;
+
+  DISALLOW_COPY_AND_ASSIGN(FieldTrialList);
+};
+
+#endif  // BASE_FIELD_TRIAL_H_
diff --git a/base/field_trial_unittest.cc b/base/field_trial_unittest.cc
new file mode 100644
index 0000000..3d8c0e2
--- /dev/null
+++ b/base/field_trial_unittest.cc
@@ -0,0 +1,74 @@
+// Copyright (c) 2006-2008 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// Test of FieldTrial class
+
+#include "base/field_trial.h"
+
+#include "base/logging.h"
+#include "testing/gtest/include/gtest/gtest.h"
+
+class FieldTrialTest : public testing::Test {
+ public:
+  FieldTrialTest() : trial_list_() { }
+  ~FieldTrialTest() { FieldTrialList::ResetConstructorCountForTestingOnly(); }
+
+ private:
+  FieldTrialList trial_list_;
+};
+
+// Test registration, and also check that destructors are called for trials
+// (and that Purify doesn't catch us leaking).
+TEST_F(FieldTrialTest, Registration) {
+  const wchar_t* name1 = L"name 1 test";
+  const wchar_t* name2 = L"name 2 test";
+  EXPECT_FALSE(FieldTrialList::Find(name1));
+  EXPECT_FALSE(FieldTrialList::Find(name2));
+
+  FieldTrial* trial1 = new FieldTrial(name1, 0.7);
+
+  EXPECT_EQ(trial1, FieldTrialList::Find(name1));
+  EXPECT_FALSE(FieldTrialList::Find(name2));
+
+  FieldTrial* trial2 = new FieldTrial(name2, 0.7);
+
+  EXPECT_EQ(trial1, FieldTrialList::Find(name1));
+  EXPECT_EQ(trial2, FieldTrialList::Find(name2));
+  // Note: FieldTrialList should delete the objects at shutdown.
+}
+
+TEST_F(FieldTrialTest, AbsoluteProbabilities) {
+  wchar_t always_true[] = L" always true";
+  wchar_t always_false[] = L" always false";
+  for (int i = 1; i < 250; ++i) {
+    // Try lots of names, by changing the first character of the name.
+    always_true[0] = i;
+    always_false[0] = i;
+    FieldTrial* trial_true = new FieldTrial(always_true, 1.0);
+    EXPECT_TRUE(trial_true->boolean_value());
+    FieldTrial* trial_false = new FieldTrial(always_false, 0.0);
+    EXPECT_FALSE(trial_false->boolean_value());
+  }
+}
+
+TEST_F(FieldTrialTest, MiddleProbabalities) {
+  wchar_t name[] = L" same name";
+  bool false_event_seen = false;
+  bool true_event_seen = false;
+  for (int i = 1; i < 250; ++i) {
+    name[0] = i;
+    FieldTrial* trial = new FieldTrial(name, 0.5);
+    if (trial->boolean_value()) {
+      true_event_seen = true;
+    } else {
+      false_event_seen = true;
+    }
+    if (false_event_seen && true_event_seen)
+      return;  // Successful test!!!
+  }
+  // Very surprising to get here. Probability should be around 1 in 2 ** 250.
+  // One of the following will fail.
+  EXPECT_TRUE(false_event_seen);
+  EXPECT_TRUE(true_event_seen);
+}
diff --git a/chrome/browser/browser.vcproj b/chrome/browser/browser.vcproj
index e2a5f7e..2edc607 100644
--- a/chrome/browser/browser.vcproj
+++ b/chrome/browser/browser.vcproj
@@ -2103,6 +2103,14 @@
 			</File>
 		</Filter>
 		<File
+			RelativePath=".\browser_trial.cc"
+			>
+		</File>
+		<File
+			RelativePath=".\browser_trial.h"
+			>
+		</File>
+		<File
 			RelativePath=".\cert_store.cc"
 			>
 		</File>
diff --git a/chrome/browser/browser_main.cc b/chrome/browser/browser_main.cc
index 5171c43..104a93f 100644
--- a/chrome/browser/browser_main.cc
+++ b/chrome/browser/browser_main.cc
@@ -24,6 +24,7 @@
 #include "chrome/browser/browser_prefs.h"
 #include "chrome/browser/browser_process_impl.h"
 #include "chrome/browser/browser_shutdown.h"
+#include "chrome/browser/browser_trial.h"
 #include "chrome/browser/cert_store.h"
 #include "chrome/browser/dom_ui/chrome_url_data_manager.h"
 #include "chrome/browser/first_run.h"
@@ -238,7 +239,7 @@ bool CheckMachineLevelInstall() {
     std::transform(user_exe_path.begin(), user_exe_path.end(),
                    user_exe_path.begin(), tolower);
     if (exe == user_exe_path) {
-      const std::wstring text = 
+      const std::wstring text =
           l10n_util::GetString(IDS_MACHINE_LEVEL_INSTALL_CONFLICT);
       const std::wstring caption = l10n_util::GetString(IDS_PRODUCT_NAME);
       const UINT flags = MB_OK | MB_ICONERROR | MB_TOPMOST;
@@ -280,6 +281,9 @@ int BrowserMain(CommandLine &parsed_command_line, int show_command,
 
   MessageLoop main_message_loop(MessageLoop::TYPE_UI);
 
+  // Initialize statistical testing infrastructure.
+  FieldTrialList field_trial;
+
   std::wstring app_name = chrome::kBrowserAppName;
   std::string thread_name_string = WideToASCII(app_name + L"_BrowserMain");
 
@@ -320,7 +324,7 @@ int BrowserMain(CommandLine &parsed_command_line, int show_command,
 
   bool is_first_run = FirstRun::IsChromeFirstRun() ||
       parsed_command_line.HasSwitch(switches::kFirstRun);
-  bool first_run_ui_bypass = false; 
+  bool first_run_ui_bypass = false;
 
   // Initialize ResourceBundle which handles files loaded from external
   // sources. This has to be done before uninstall code path and before prefs
@@ -339,7 +343,7 @@ int BrowserMain(CommandLine &parsed_command_line, int show_command,
       local_state->SetBoolean(prefs::kMetricsReportingEnabled, true);
     // On first run, we  need to process the master preferences before the
     // browser's profile_manager object is created.
-    FirstRun::MasterPrefResult master_pref_res = 
+    FirstRun::MasterPrefResult master_pref_res =
         FirstRun::ProcessMasterPreferences(user_data_dir, std::wstring());
     first_run_ui_bypass =
         (master_pref_res == FirstRun::MASTER_PROFILE_NO_FIRST_RUN_UI);
@@ -505,7 +509,7 @@ int BrowserMain(CommandLine &parsed_command_line, int show_command,
   ShellIntegration::VerifyInstallation();
 
   browser_process->InitBrokerServices(broker_services);
-  
+
   // In unittest mode, this will do nothing.  In normal mode, this will create
   // the global GoogleURLTracker instance, which will promptly go to sleep for
   // five seconds (to avoid slowing startup), and wake up afterwards to see if
@@ -544,10 +548,9 @@ int BrowserMain(CommandLine &parsed_command_line, int show_command,
     }
     metrics = browser_process->metrics_service();
     DCHECK(metrics);
-    
-    // If we're testing then we don't care what the user 
-    // preference is, we turn on recording, but not reporting, otherwise tests
-    // fail.
+
+    // If we're testing then we don't care what the user preference is, we turn
+    // on recording, but not reporting, otherwise tests fail.
     if (parsed_command_line.HasSwitch(switches::kMetricsRecordingOnly)) {
       metrics->StartRecordingOnly();
     } else {
diff --git a/chrome/browser/browser_process_impl.cc b/chrome/browser/browser_process_impl.cc
index 9ff0b7b..d4aabe7 100644
--- a/chrome/browser/browser_process_impl.cc
+++ b/chrome/browser/browser_process_impl.cc
@@ -5,9 +5,10 @@
 #include "chrome/browser/browser_process_impl.h"
 
 #include "base/command_line.h"
-#include "base/thread.h"
 #include "base/path_service.h"
+#include "base/thread.h"
 #include "chrome/browser/automation/automation_provider_list.h"
+#include "chrome/browser/browser_trial.h"
 #include "chrome/browser/chrome_thread.h"
 #include "chrome/browser/download/download_file.h"
 #include "chrome/browser/download/save_file_manager.h"
@@ -115,6 +116,16 @@ BrowserProcessImpl::BrowserProcessImpl(CommandLine& command_line)
       else if (model == L"medium")
         memory_model_ = MEDIUM_MEMORY_MODEL;
     }
+  } else {
+    // Randomly choose what memory model to use.
+    const double probability = 0.5;
+    FieldTrial* trial(new FieldTrial(BrowserTrial::kMemoryModelFieldTrial,
+                                     probability));
+    DCHECK(FieldTrialList::Find(BrowserTrial::kMemoryModelFieldTrial) == trial);
+    if (trial->boolean_value())
+      memory_model_ = HIGH_MEMORY_MODEL;
+    else
+      memory_model_ = MEDIUM_MEMORY_MODEL;
   }
 
   suspend_controller_ = new SuspendController();
diff --git a/chrome/browser/browser_trial.cc b/chrome/browser/browser_trial.cc
new file mode 100644
index 0000000..16bfb18
--- /dev/null
+++ b/chrome/browser/browser_trial.cc
@@ -0,0 +1,8 @@
+// Copyright (c) 2006-2008 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "chrome/browser/browser_trial.h"
+
+// A test to determine the impact of using HIGH vs MEDIUM memory models.
+const wchar_t* BrowserTrial::kMemoryModelFieldTrial = L"memory_model";
diff --git a/chrome/browser/browser_trial.h b/chrome/browser/browser_trial.h
new file mode 100644
index 0000000..1752eba
--- /dev/null
+++ b/chrome/browser/browser_trial.h
@@ -0,0 +1,22 @@
+// Copyright (c) 2006-2008 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// BrowserTrial contains methods specific to field trials (using FieldTrial
+// functionality) for browser tests.
+
+#ifndef CHROME_BROWSER_BROWSER_TRIAL_H_
+#define CHROME_BROWSER_BROWSER_TRIAL_H_
+
+#include "base/field_trial.h"
+
+// Currently we use this as a name space, to hold static shared constants which
+// define current and past trials.
+class BrowserTrial {
+ public:
+  static const wchar_t* kMemoryModelFieldTrial;
+ private:
+  DISALLOW_COPY_AND_ASSIGN(BrowserTrial);
+};
+
+#endif  // CHROME_BROWSER_BROWSER_TRIAL_H_
diff --git a/chrome/browser/render_widget_host_view_win.cc b/chrome/browser/render_widget_host_view_win.cc
index 3788795..eef290f 100644
--- a/chrome/browser/render_widget_host_view_win.cc
+++ b/chrome/browser/render_widget_host_view_win.cc
@@ -11,6 +11,7 @@
 #include "base/win_util.h"
 #include "chrome/browser/browser_accessibility.h"
 #include "chrome/browser/browser_accessibility_manager.h"
+#include "chrome/browser/browser_trial.h"
 #include "chrome/browser/render_process_host.h"
 // TODO(beng): (Cleanup) we should not need to include this file... see comment
 //             in |DidBecomeSelected|.
@@ -433,7 +434,21 @@ void RenderWidgetHostViewWin::OnPaint(HDC dc) {
     }
     if (!whiteout_start_time_.is_null()) {
       TimeDelta whiteout_duration = TimeTicks::Now() - whiteout_start_time_;
-      UMA_HISTOGRAM_TIMES(L"MPArch.RWHH_WhiteoutDuration", whiteout_duration);
+
+      // If field trial is active, report results in special histogram.
+      static scoped_refptr<FieldTrial> trial(
+          FieldTrialList::Find(BrowserTrial::kMemoryModelFieldTrial));
+      if (trial.get()) {
+        if (trial->boolean_value())
+          UMA_HISTOGRAM_TIMES(L"MPArch.RWHH_WhiteoutDuration_trial_high_memory",
+                              whiteout_duration);
+        else
+          UMA_HISTOGRAM_TIMES(L"MPArch.RWHH_WhiteoutDuration_trial_med_memory",
+                              whiteout_duration);
+      } else {
+        UMA_HISTOGRAM_TIMES(L"MPArch.RWHH_WhiteoutDuration", whiteout_duration);
+      }
+
       // Reset the start time to 0 so that we start recording again the next
       // time the backing store is NULL...
       whiteout_start_time_ = TimeTicks();