[Telemetry] Add a WillRunPageSet()/DidRunPageSet() API.

This is useful for benchmarks to call immediately prior to and after
running a page_set. It will be used by the page cyclers to record memory and IO
statistics which are a delta of before and after the entire page set runs.

This involved adding the concept of perf results that are not associated with a
particular page.

BUG=None
TEST=tools/telemetry/run_tests --browser=system page_benchmark_results_unittest
NOTRY=True


Review URL: https://chromiumcodereview.appspot.com/13488005

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@192276 0039d316-1c4b-4281-b951-d872f2087c98

4 files changed, 68 insertions, 2 deletions

diff --git a/tools/telemetry/telemetry/page/page_benchmark_results.py b/tools/telemetry/telemetry/page/page_benchmark_results.py
index 6a97346..b13e2b9 100644
--- a/tools/telemetry/telemetry/page/page_benchmark_results.py
+++ b/tools/telemetry/telemetry/page/page_benchmark_results.py
@@ -47,6 +47,7 @@ class PageBenchmarkResults(page_test.PageTestResults):
   def __init__(self):
     super(PageBenchmarkResults, self).__init__()
     self._page_results = []
+    self._overall_results = []
 
     self._all_measurements_that_have_been_seen = {}
 
@@ -75,6 +76,18 @@ class PageBenchmarkResults(page_test.PageTestResults):
     return self._all_measurements_that_have_been_seen
 
   def Add(self, trace_name, units, value, chart_name=None, data_type='default'):
+    value = self._GetPageBenchmarkValue(trace_name, units, value, chart_name,
+                                        data_type)
+    self._values_for_current_page.AddValue(value)
+
+  def AddSummary(self, trace_name, units, value, chart_name=None,
+                 data_type='default'):
+    value = self._GetPageBenchmarkValue(trace_name, units, value, chart_name,
+                                        data_type)
+    self._overall_results.append(value)
+
+  def _GetPageBenchmarkValue(self, trace_name, units, value, chart_name,
+                             data_type):
     value = page_benchmark_value.PageBenchmarkValue(
         trace_name, units, value, chart_name, data_type)
     measurement_name = value.measurement_name
@@ -94,8 +107,7 @@ class PageBenchmarkResults(page_test.PageTestResults):
       self._all_measurements_that_have_been_seen[measurement_name] = {
         'units': units,
         'type': data_type}
-
-    self._values_for_current_page.AddValue(value)
+    return value
 
   def DidMeasurePage(self):
     assert self._values_for_current_page, 'Failed to call WillMeasurePage'
@@ -167,3 +179,14 @@ class PageBenchmarkResults(page_test.PageTestResults):
         if isinstance(values[0], list):
           values = list(chain.from_iterable(values))
         self._PrintPerfResult(measurement, trace, values, units, data_type)
+
+    # Output the overall results (results not associated with a page).
+    for value in self._overall_results:
+      values = value.value
+      if not isinstance(values, list):
+        values = [values]
+      measurement_name = value.chart_name
+      if not measurement_name:
+        measurement_name = value.trace_name
+      self._PrintPerfResult(measurement_name, value.trace_name,
+                            values, value.units, value.data_type)
diff --git a/tools/telemetry/telemetry/page/page_benchmark_results_unittest.py b/tools/telemetry/telemetry/page/page_benchmark_results_unittest.py
index b2edad3..e3e20bf 100644
--- a/tools/telemetry/telemetry/page/page_benchmark_results_unittest.py
+++ b/tools/telemetry/telemetry/page/page_benchmark_results_unittest.py
@@ -139,6 +139,34 @@ class PageBenchmarkResultsTest(unittest.TestCase):
       benchmark_results.results,
       expected)
 
+  def test_overall_results(self):
+    test_page_set = _MakePageSet()
+
+    benchmark_results = SummarySavingPageBenchmarkResults()
+
+    benchmark_results.AddSummary('a', 'seconds', 1)
+
+    benchmark_results.WillMeasurePage(test_page_set.pages[0])
+    benchmark_results.Add('b', 'seconds', 2)
+    benchmark_results.DidMeasurePage()
+
+    benchmark_results.WillMeasurePage(test_page_set.pages[1])
+    benchmark_results.Add('b', 'seconds', 3)
+    benchmark_results.DidMeasurePage()
+
+    benchmark_results.AddSummary('c', 'seconds', 4)
+
+    benchmark_results.PrintSummary(None)
+    expected = ['RESULT b_by_url: http___www.foo.com_= 2 seconds',
+                'RESULT b_by_url: http___www.bar.com_= 3 seconds',
+                '*RESULT b: b= [2,3] seconds\n' +
+                'Avg b: 2.500000seconds\nSd  b: 0.707107seconds',
+                '*RESULT a: a= 1 seconds',
+                '*RESULT c: c= 4 seconds']
+    self.assertEquals(
+      benchmark_results.results,
+      expected)
+
   def test_histogram(self):
     test_page_set = _MakePageSet()
 
diff --git a/tools/telemetry/telemetry/page/page_runner.py b/tools/telemetry/telemetry/page/page_runner.py
index 0dd8d52..d2c3313 100644
--- a/tools/telemetry/telemetry/page/page_runner.py
+++ b/tools/telemetry/telemetry/page/page_runner.py
@@ -140,6 +140,7 @@ class PageRunner(object):
 
     state = _RunState()
     last_archive_path = None
+    is_first_run = True
     try:
       for page in pages:
         if options.wpr_mode != wpr_modes.WPR_RECORD:
@@ -171,6 +172,10 @@ class PageRunner(object):
 
             self._WaitForThermalThrottlingIfNeeded(state.browser.platform)
 
+            if is_first_run:
+              is_first_run = False
+              test.WillRunPageSet(state.tab, results)
+
             try:
               self._RunPage(options, page, state.tab, test, results)
               self._CheckThermalThrottling(state.browser.platform)
@@ -199,6 +204,7 @@ class PageRunner(object):
             if not tries:
               logging.error('Lost connection to browser 3 times. Failing.')
               raise
+      test.DidRunPageSet(state.tab, results)
     finally:
       state.Close()
 
diff --git a/tools/telemetry/telemetry/page/page_test.py b/tools/telemetry/telemetry/page/page_test.py
index 2e9390b..482d166 100644
--- a/tools/telemetry/telemetry/page/page_test.py
+++ b/tools/telemetry/telemetry/page/page_test.py
@@ -106,6 +106,15 @@ class PageTest(object):
     """Override to customize if the test can be ran for the given page."""
     return True
 
+  def WillRunPageSet(self, tab, results):
+    """Override to do operations before the page set is navigated."""
+    pass
+
+  def DidRunPageSet(self, tab, results):
+    """Override to do operations after page set is completed, but before browser
+    is torn down."""
+    pass
+
   def WillNavigateToPage(self, page, tab):
     """Override to do operations before the page is navigated."""
     pass