Add scroll test for real world web sites.

For now it uses the same sites list as the page cycler and it is disabled.
I plan to change that.

BUG=134057
TEST=chrome/test/functional/perf.py perf.PopularSitesScrollTest.test2012Q2


Review URL: https://chromiumcodereview.appspot.com/10702032

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@145547 0039d316-1c4b-4281-b951-d872f2087c98

3 files changed, 145 insertions, 70 deletions

diff --git a/chrome/test/data/scroll/scroll.js b/chrome/test/data/scroll/scroll.js
index 305c1ea..5a9f12b 100644
--- a/chrome/test/data/scroll/scroll.js
+++ b/chrome/test/data/scroll/scroll.js
@@ -8,7 +8,7 @@
 // USAGE:
 //   1. To start the scrolling, invoke __scroll_test().
 //   2. Wait for __scrolling_complete to be true
-//   3. Read __initial_frame_times and __repeat_frame_times arrays.
+//   3. Read __frame_times array.
 
 
 // Function that sets a callback called when the next frame is rendered.
diff --git a/chrome/test/functional/PYAUTO_TESTS b/chrome/test/functional/PYAUTO_TESTS
index 6ab1ea0..15ab8cf 100644
--- a/chrome/test/functional/PYAUTO_TESTS
+++ b/chrome/test/functional/PYAUTO_TESTS
@@ -629,6 +629,7 @@
       # ===========================
       # Invoked outside of the pyauto_tests framework.
       '-perf.BenchmarkPerfTest.testSpaceport',
+      '-perf.PopularSitesScrollTest.test2012Q2',
     ],
     'win': [
       # ==================================================
diff --git a/chrome/test/functional/perf.py b/chrome/test/functional/perf.py
index b3a5a99..3c45980 100755
--- a/chrome/test/functional/perf.py
+++ b/chrome/test/functional/perf.py
@@ -58,9 +58,7 @@ from youtube import YoutubeTestHelper
 def StandardDeviation(values):
   """Returns the standard deviation of |values|."""
   avg = Mean(values)
-  if not avg:
-    return None
-  if len(values) < 2:
+  if len(values) < 2 or not avg:
     return 0.0
   temp_vals = [math.pow(x - avg, 2) for x in values]
   return math.sqrt(sum(temp_vals) / (len(temp_vals) - 1))
@@ -1619,6 +1617,9 @@ class FrameTimes(object):
       return 0
     return int(1000.0 / avg)
 
+  def GetMeanFrameTime(self):
+    return Mean(self._frame_times)
+
   def GetPercentBelow60Fps(self):
     if not self._frame_times:
       return 0
@@ -1633,24 +1634,71 @@ class ScrollResults(object):
 
   def __init__(self, first_paint_seconds, frame_times_lists):
     assert len(frame_times_lists) == 2, 'Expecting initial and repeat times'
-    self.first_paint_times = []
+    self.first_paint_time = 1000.0 * first_paint_seconds
     self.initial_frame_times = FrameTimes(frame_times_lists[0])
     self.repeat_frame_times = FrameTimes(frame_times_lists[1])
 
 
-class ScrollTest(BasePerfTest):
-  """Tests to measure scrolling performance."""
+class BaseScrollTest(BasePerfTest):
+  """Base class for tests measuring scrolling performance."""
 
-  def ExtraChromeFlags(self):
-    """Ensures Chrome is launched with custom flags.
+  def setUp(self):
+    """Performs necessary setup work before running each test."""
+    super(BaseScrollTest, self).setUp()
+    scroll_file = os.path.join(self.DataDir(), 'scroll', 'scroll.js')
+    with open(scroll_file) as f:
+      self._scroll_text = f.read()
+
+  def RunSingleInvocation(self, url, setup_js=''):
+    """Runs a single invocation of the scroll test.
+
+    Args:
+      url: The string url for the webpage on which to run the scroll test.
+      setup_js: String representing additional Javascript setup code to execute
+          in the webpage immediately before running the scroll test.
 
     Returns:
-      A list of extra flags to pass to Chrome when it is launched.
+      Instance of ScrollResults.
     """
-    # Extra flag needed by scroll performance tests.
-    return super(ScrollTest, self).ExtraChromeFlags() + ['--disable-gpu-vsync']
+    self.assertTrue(self.AppendTab(pyauto.GURL(url)),
+                    msg='Failed to append tab for webpage.')
+
+    js = """
+        %s
+        %s
+        __scroll_test();
+        window.domAutomationController.send('done');
+    """ % (self._scroll_text, setup_js)
+    self.ExecuteJavascript(js, tab_index=1)
+
+    # Poll the webpage until the test is complete.
+    def IsTestComplete():
+      done_js = """
+        if (__scrolling_complete)
+          window.domAutomationController.send('complete');
+        else
+          window.domAutomationController.send('incomplete');
+      """
+      return self.ExecuteJavascript(done_js, tab_index=1) == 'complete'
 
-  def _RunScrollTest(self, url, description, graph_name, setup_js=''):
+    self.assertTrue(
+        self.WaitUntil(IsTestComplete, timeout=300, expect_retval=True,
+                       retry_sleep=1),
+        msg='Timed out when waiting for scrolling tests to complete.')
+
+    # Get the scroll test results from the webpage.
+    results_js = """
+      window.domAutomationController.send(JSON.stringify({
+          'first_paint_time': chrome.loadTimes().firstPaintTime -
+                              chrome.loadTimes().requestTime,
+          'frame_times': __frame_times,
+      }));
+    """
+    results = eval(self.ExecuteJavascript(results_js, tab_index=1))
+    self.GetBrowserWindow(0).GetTab(1).Close(True)
+    return ScrollResults(results['first_paint_time'], results['frame_times'])
+
+  def RunScrollTest(self, url, description, graph_name, setup_js=''):
     """Runs a scroll performance test on the specified webpage.
 
     Args:
@@ -1661,89 +1709,114 @@ class ScrollTest(BasePerfTest):
       setup_js: String representing additional Javascript setup code to execute
           in the webpage immediately before running the scroll test.
     """
-    scroll_file = os.path.join(self.DataDir(), 'scroll', 'scroll.js')
-    with open(scroll_file) as f:
-      scroll_text = f.read()
-
-    def _RunSingleInvocation(url, scroll_text):
-      """Runs a single invocation of the scroll test.
-
-      Returns:
-        Instance of ScrollResults.
-      """
-      self.assertTrue(self.AppendTab(pyauto.GURL(url)),
-                      msg='Failed to append tab for webpage.')
-
-      js = """
-          %s
-          %s
-          __scroll_test();
-          window.domAutomationController.send('done');
-      """ % (scroll_text, setup_js)
-      self.ExecuteJavascript(js, tab_index=1)
-
-      # Poll the webpage until the test is complete.
-      def IsTestDone():
-        done_js = """
-          if (__scrolling_complete)
-            window.domAutomationController.send('true');
-          else
-            window.domAutomationController.send('false');
-        """
-        return self.ExecuteJavascript(done_js, tab_index=1) == 'true'
-
-      self.assertTrue(
-          self.WaitUntil(IsTestDone, timeout=300, expect_retval=True,
-                         retry_sleep=1),
-          msg='Timed out when waiting for scrolling tests to complete.')
-
-      # Get the scroll test results from the webpage.
-      results_js = """
-        window.domAutomationController.send(JSON.stringify({
-            'first_paint_time': chrome.loadTimes().firstPaintTime -
-                                chrome.loadTimes().requestTime,
-            'frame_times': __frame_times,
-        }));
-      """
-      results = eval(self.ExecuteJavascript(results_js, tab_index=1))
-      self.GetBrowserWindow(0).GetTab(1).Close(True)
-      return ScrollResults(results['first_paint_time'], results['frame_times'])
-
-    fps_vals = []
+    results = []
     for iteration in range(self._num_iterations + 1):
-      result = _RunSingleInvocation(url, scroll_text)
+      result = self.RunSingleInvocation(url, setup_js)
       # Ignore the first iteration.
       if iteration:
         fps = result.repeat_frame_times.GetFps()
+        assert fps, '%s did not scroll' % url
         logging.info('Iteration %d of %d: %f fps', iteration,
                      self._num_iterations, fps)
-        fps_vals.append(fps)
+        results.append(result)
+    self._PrintSummaryResults(
+        description, [r.repeat_frame_times.GetFps() for r in results],
+        'FPS', graph_name)
 
-    self._PrintSummaryResults(description, fps_vals, 'FPS', graph_name)
+
+class PopularSitesScrollTest(BaseScrollTest):
+  """Measures scrolling performance on recorded versions of popular sites."""
+
+  def ExtraChromeFlags(self):
+    """Ensures Chrome is launched with custom flags.
+
+    Returns:
+      A list of extra flags to pass to Chrome when it is launched.
+    """
+    return super(PopularSitesScrollTest,
+                 self).ExtraChromeFlags() + WebPageReplay.CHROME_FLAGS
+
+  def _GetUrlList(self, test_name):
+    """Returns list of recorded sites."""
+    with open(WebPageReplay.Path('page_sets', test_name=test_name)) as f:
+      sites_text = f.read()
+    js = """
+      %s
+      window.domAutomationController.send(JSON.stringify(pageSets));
+    """ % sites_text
+    page_sets = eval(self.ExecuteJavascript(js))
+    return list(itertools.chain(*page_sets))[1:]  # Skip first.
+
+  def _PrintScrollResults(self, results):
+    self._PrintSummaryResults(
+        'initial', [r.initial_frame_times.GetMeanFrameTime() for r in results],
+        'ms', 'FrameTimes')
+    self._PrintSummaryResults(
+        'repeat', [r.repeat_frame_times.GetMeanFrameTime() for r in results],
+        'ms', 'FrameTimes')
+    self._PrintSummaryResults(
+        'initial',
+        [r.initial_frame_times.GetPercentBelow60Fps() for r in results],
+        'percent', 'PercentBelow60FPS')
+    self._PrintSummaryResults(
+        'repeat',
+        [r.repeat_frame_times.GetPercentBelow60Fps() for r in results],
+        'percent', 'PercentBelow60FPS')
+    self._PrintSummaryResults(
+        'first_paint_time', [r.first_paint_time for r in results],
+        'ms', 'FirstPaintTime')
+
+  def test2012Q2(self):
+    test_name = '2012Q2'
+    urls = self._GetUrlList(test_name)
+    results = []
+    with WebPageReplay().GetReplayServer(test_name):
+      for iteration in range(self._num_iterations):
+        for url in urls:
+          result = self.RunSingleInvocation(url)
+          fps = result.initial_frame_times.GetFps()
+          assert fps, '%s did not scroll' % url
+          logging.info('Iteration %d of %d: %f fps', iteration,
+                       self._num_iterations, fps)
+          results.append(result)
+    self._PrintScrollResults(results)
+
+
+class ScrollTest(BaseScrollTest):
+  """Tests to measure scrolling performance."""
+
+  def ExtraChromeFlags(self):
+    """Ensures Chrome is launched with custom flags.
+
+    Returns:
+      A list of extra flags to pass to Chrome when it is launched.
+    """
+    # Extra flag needed by scroll performance tests.
+    return super(ScrollTest, self).ExtraChromeFlags() + ['--disable-gpu-vsync']
 
   def testBlankPageScroll(self):
     """Runs the scroll test on a blank page."""
-    self._RunScrollTest(
+    self.RunScrollTest(
         self.GetFileURLForDataPath('scroll', 'blank.html'), 'ScrollBlankPage',
         'scroll_fps')
 
   def testTextScroll(self):
     """Runs the scroll test on a text-filled page."""
-    self._RunScrollTest(
+    self.RunScrollTest(
         self.GetFileURLForDataPath('scroll', 'text.html'), 'ScrollTextPage',
         'scroll_fps')
 
   def testGooglePlusScroll(self):
     """Runs the scroll test on a Google Plus anonymized page."""
-    self._RunScrollTest(
+    self.RunScrollTest(
         self.GetFileURLForDataPath('scroll', 'plus.html'),
         'ScrollGooglePlusPage', 'scroll_fps')
 
   def testGmailScroll(self):
     """Runs the scroll test using the live Gmail site."""
     self._LoginToGoogleAccount(account_key='test_google_account_gmail')
-    self._RunScrollTest('http://www.gmail.com', 'ScrollGmail', 'scroll_fps',
-                        setup_js='__is_gmail_test = true;')
+    self.RunScrollTest('http://www.gmail.com', 'ScrollGmail', 'scroll_fps',
+                       setup_js='__is_gmail_test = true;')
 
 
 class FlashTest(BasePerfTest):
@@ -2114,6 +2187,7 @@ class WebPageReplay(object):
   """
   _PATHS = {
       'archive':    'src/data/page_cycler/webpagereplay/{test_name}.wpr',
+      'page_sets':  'src/tools/page_cycler/webpagereplay/tests/{test_name}.js',
       'start_page': 'src/tools/page_cycler/webpagereplay/start.html',
       'extension':  'src/tools/page_cycler/webpagereplay/extension',
       'replay':     'src/third_party/webpagereplay',