[chromedriver] Add some simple perf benchmarks.

These always pass and just output the results for now.
BUG=chromedriver:10
NOTRY=true

Review URL: https://chromiumcodereview.appspot.com/15979030

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@204037 0039d316-1c4b-4281-b951-d872f2087c98

2 files changed, 121 insertions, 19 deletions

diff --git a/chrome/test/chromedriver/run_all_tests.py b/chrome/test/chromedriver/run_all_tests.py
index 50e3870..5cbf7e4 100755
--- a/chrome/test/chromedriver/run_all_tests.py
+++ b/chrome/test/chromedriver/run_all_tests.py
@@ -34,13 +34,19 @@ def _AddToolsToSystemPathForWindows():
   os.environ['PATH'] = os.pathsep.join(paths) + os.pathsep + os.environ['PATH']
 
 
-def _GenerateTestCommand(script, chromedriver, chrome=None,
-                         chrome_version=None, android_package=None):
+def _GenerateTestCommand(script,
+                         chromedriver,
+                         ref_chromedriver=None,
+                         chrome=None,
+                         chrome_version=None,
+                         android_package=None):
   cmd = [
       sys.executable,
       os.path.join(_THIS_DIR, script),
       '--chromedriver=' + chromedriver,
   ]
+  if ref_chromedriver:
+    cmd.append('--reference-chromedriver=' + ref_chromedriver)
   if chrome:
     cmd.append('--chrome=' + chrome)
   if chrome_version:
@@ -51,15 +57,20 @@ def _GenerateTestCommand(script, chromedriver, chrome=None,
   return cmd
 
 
-def RunPythonTests(chromedriver, chrome=None, chrome_version=None,
+def RunPythonTests(chromedriver, ref_chromedriver,
+                   chrome=None, chrome_version=None,
                    chrome_version_name=None, android_package=None):
   version_info = ''
   if chrome_version_name:
     version_info = '(v%s)' % chrome_version_name
   util.MarkBuildStepStart('python_tests%s' % version_info)
   code = util.RunCommand(
-      _GenerateTestCommand('run_py_tests.py', chromedriver, chrome,
-                           chrome_version, android_package))
+      _GenerateTestCommand('run_py_tests.py',
+                           chromedriver,
+                           ref_chromedriver=ref_chromedriver,
+                           chrome=chrome,
+                           chrome_version=chrome_version,
+                           android_package=android_package))
   if code:
     util.MarkBuildStepError()
   return code
@@ -72,8 +83,12 @@ def RunJavaTests(chromedriver, chrome=None, chrome_version=None,
     version_info = '(v%s)' % chrome_version_name
   util.MarkBuildStepStart('java_tests%s' % version_info)
   code = util.RunCommand(
-      _GenerateTestCommand('run_java_tests.py', chromedriver, chrome,
-                           chrome_version, android_package))
+      _GenerateTestCommand('run_java_tests.py',
+                           chromedriver,
+                           ref_chromedriver=None,
+                           chrome=chrome,
+                           chrome_version=chrome_version,
+                           android_package=android_package))
   if code:
     util.MarkBuildStepError()
   return code
@@ -100,11 +115,11 @@ def main():
            'Notice: this option only applies to desktop.')
   options, _ = parser.parse_args()
 
-  postfix = ''
+  exe_postfix = ''
   if util.IsWindows():
-    postfix = '.exe'
-  cpp_tests_name = 'chromedriver2_tests' + postfix
-  server_name = 'chromedriver2_server' + postfix
+    exe_postfix = '.exe'
+  cpp_tests_name = 'chromedriver2_tests' + exe_postfix
+  server_name = 'chromedriver2_server' + exe_postfix
 
   required_build_outputs = [server_name]
   if not options.android_package:
@@ -112,7 +127,12 @@ def main():
   build_dir = chrome_paths.GetBuildDir(required_build_outputs)
   print 'Using build outputs from', build_dir
 
-  chromedriver_server = os.path.join(build_dir, server_name)
+  chromedriver = os.path.join(build_dir, server_name)
+  ref_chromedriver = os.path.join(
+      chrome_paths.GetSrc(),
+      'chrome', 'test', 'chromedriver', 'third_party', 'java_tests',
+      'reference_builds',
+      'chromedriver_%s%s' % (util.GetPlatformName(), exe_postfix))
 
   if util.IsLinux():
     # Set LD_LIBRARY_PATH to enable successful loading of shared object files,
@@ -124,9 +144,10 @@ def main():
 
   if options.android_package:
     os.environ['PATH'] += os.pathsep + os.path.join(_THIS_DIR, 'chrome')
-    code1 = RunPythonTests(chromedriver_server,
+    code1 = RunPythonTests(chromedriver,
+                           ref_chromedriver,
                            android_package=options.android_package)
-    code2 = RunJavaTests(chromedriver_server,
+    code2 = RunJavaTests(chromedriver,
                          android_package=options.android_package)
     return code1 or code2
   else:
@@ -148,10 +169,12 @@ def main():
       chrome_path = archive.DownloadChrome(version[1],
                                            util.MakeTempDir(),
                                            download_site)
-      code1 = RunPythonTests(chromedriver_server, chrome=chrome_path,
+      code1 = RunPythonTests(chromedriver,
+                             ref_chromedriver,
+                             chrome=chrome_path,
                              chrome_version=version[0],
                              chrome_version_name=version_name)
-      code2 = RunJavaTests(chromedriver_server, chrome=chrome_path,
+      code2 = RunJavaTests(chromedriver, chrome=chrome_path,
                            chrome_version=version[0],
                            chrome_version_name=version_name)
       code = code or code1 or code2
diff --git a/chrome/test/chromedriver/run_py_tests.py b/chrome/test/chromedriver/run_py_tests.py
index 5781188..9ec266e 100755
--- a/chrome/test/chromedriver/run_py_tests.py
+++ b/chrome/test/chromedriver/run_py_tests.py
@@ -96,6 +96,10 @@ _ANDROID_NEGATIVE_FILTER['com.google.android.apps.chrome'] = (
         'ChromeDriverTest.testWindowSize',
         'ChromeDriverTest.testWindowMaximize',
         'ChromeLogPathCapabilityTest.testChromeLogPath',
+        # Don't enable perf testing on Android yet.
+        'PerfTest.testSessionStartTime',
+        'PerfTest.testSessionStopTime',
+        'PerfTest.testColdExecuteScript',
     ]
 )
 _ANDROID_NEGATIVE_FILTER['org.chromium.chrome.testshell'] = (
@@ -114,11 +118,13 @@ class ChromeDriverBaseTest(unittest.TestCase):
     for driver in self._drivers:
       try:
         driver.Quit()
-      except chromedriver.ChromeDriverException:
+      except:
         pass
 
-  def CreateDriver(self, **kwargs):
-    driver = chromedriver.ChromeDriver(_CHROMEDRIVER_SERVER_URL,
+  def CreateDriver(self, server_url=None, **kwargs):
+    if server_url is None:
+      server_url = _CHROMEDRIVER_SERVER_URL
+    driver = chromedriver.ChromeDriver(server_url,
                                        chrome_binary=_CHROME_BINARY,
                                        android_package=_ANDROID_PACKAGE,
                                        **kwargs)
@@ -608,12 +614,82 @@ class ChromeLogPathCapabilityTest(ChromeDriverBaseTest):
     self.assertTrue(self.LOG_MESSAGE in open(tmp_log_path.name).read())
 
 
+class PerfTest(ChromeDriverBaseTest):
+  """Tests for ChromeDriver perf."""
+  def setUp(self):
+    self.assertTrue(_REFERENCE_CHROMEDRIVER is not None,
+                    'must supply a reference-chromedriver arg')
+
+  def _RunDriverPerfTest(self, name, test_func):
+    """Runs a perf test comparing a reference and new ChromeDriver server.
+
+    Args:
+      name: The name of the perf test.
+      test_func: Called with the server url to perform the test action. Must
+                 return the time elapsed.
+    """
+    class Results(object):
+      ref = []
+      new = []
+
+    ref_server = chromedriver.Server(_REFERENCE_CHROMEDRIVER)
+    results = Results()
+    result_url_pairs = zip([results.new, results.ref],
+                           [_CHROMEDRIVER_SERVER_URL, ref_server.GetUrl()])
+    for iteration in range(30):
+      for result, url in result_url_pairs:
+        result += [test_func(url)]
+      # Reverse the order for the next run.
+      result_url_pairs = result_url_pairs[::-1]
+
+    def PrintResult(build, result):
+      mean = sum(result) / len(result)
+      avg_dev = sum([abs(sample - mean) for sample in result]) / len(result)
+      print 'perf result', build, name, mean, avg_dev, result
+      util.AddBuildStepText(u'%s %s: %.3f\u00b1%.3f' % (
+          build, name, mean, avg_dev))
+
+    # Discard first result, which may be off due to cold start.
+    PrintResult('new', results.new[1:])
+    PrintResult('ref', results.ref[1:])
+
+  def testSessionStartTime(self):
+    def Run(url):
+      start = time.time()
+      driver = self.CreateDriver(url)
+      end = time.time()
+      driver.Quit()
+      return end - start
+    self._RunDriverPerfTest('session start', Run)
+
+  def testSessionStopTime(self):
+    def Run(url):
+      driver = self.CreateDriver(url)
+      start = time.time()
+      driver.Quit()
+      end = time.time()
+      return end - start
+    self._RunDriverPerfTest('session stop', Run)
+
+  def testColdExecuteScript(self):
+    def Run(url):
+      driver = self.CreateDriver(url)
+      start = time.time()
+      driver.ExecuteScript('return 1')
+      end = time.time()
+      driver.Quit()
+      return end - start
+    self._RunDriverPerfTest('cold exe js', Run)
+
 if __name__ == '__main__':
   parser = optparse.OptionParser()
   parser.add_option(
       '', '--chromedriver',
       help='Path to chromedriver server (REQUIRED!)')
   parser.add_option(
+      '', '--reference-chromedriver',
+      help='Path to the reference chromedriver server')
+  parser.add_option(
       '', '--chrome', help='Path to a build of the chrome binary')
   parser.add_option(
       '', '--chrome-version', default='HEAD',
@@ -634,6 +710,9 @@ if __name__ == '__main__':
   global _CHROMEDRIVER_SERVER_URL
   _CHROMEDRIVER_SERVER_URL = server.GetUrl()
 
+  global _REFERENCE_CHROMEDRIVER
+  _REFERENCE_CHROMEDRIVER = options.reference_chromedriver
+
   global _CHROME_BINARY
   if options.chrome:
     _CHROME_BINARY = os.path.abspath(options.chrome)