telemetry: Allow pages that are expected to fail to crash or time out

If a page is expected to fail and while we are running it the browser
or the tab crashes, we fail to log in, collect profiling data or the
test times out, count the test as successful. This is so that test
expectations can be used when a problem with a test manifests as one of
the aforementioned symptoms.

BUG=272080
TEST=PageRunnerTests.testHandlingOfCrashedTabWithExpectedFailure

Review URL: https://chromiumcodereview.appspot.com/22824011

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@217545 0039d316-1c4b-4281-b951-d872f2087c98

2 files changed, 29 insertions, 4 deletions

diff --git a/tools/telemetry/telemetry/page/page_runner.py b/tools/telemetry/telemetry/page/page_runner.py
index 84ca2c0..89ab311 100644
--- a/tools/telemetry/telemetry/page/page_runner.py
+++ b/tools/telemetry/telemetry/page/page_runner.py
@@ -373,6 +373,14 @@ def _RunPage(test, page, state, expectation, results, options):
   page_state = PageState()
   tab = state.tab
 
+  def ProcessError():
+    logging.error('%s:\n%s', page.url, traceback.format_exc())
+    if expectation == 'fail':
+      logging.info('Error was expected\n')
+      results.AddSuccess(page)
+    else:
+      results.AddError(page, sys.exc_info())
+
   try:
     page_state.PreparePage(page, tab, test)
     if state.repeat_state.ShouldNavigate(options.skip_navigate_on_repeat):
@@ -388,11 +396,9 @@ def _RunPage(test, page, state, expectation, results, options):
       results.AddFailure(page, sys.exc_info())
   except (util.TimeoutException, exceptions.LoginException,
           exceptions.ProfilingException):
-    logging.error('%s:\n%s', page.url, traceback.format_exc())
-    results.AddError(page, sys.exc_info())
+    ProcessError()
   except (exceptions.TabCrashException, exceptions.BrowserGoneException):
-    logging.error('%s:\n%s', page.url, traceback.format_exc())
-    results.AddError(page, sys.exc_info())
+    ProcessError()
     # Run() catches these exceptions to relaunch the tab/browser, so re-raise.
     raise
   except Exception:
diff --git a/tools/telemetry/telemetry/page/page_runner_unittest.py b/tools/telemetry/telemetry/page/page_runner_unittest.py
index 2ff1cfc..815683b 100644
--- a/tools/telemetry/telemetry/page/page_runner_unittest.py
+++ b/tools/telemetry/telemetry/page/page_runner_unittest.py
@@ -59,8 +59,27 @@ class PageRunnerTests(unittest.TestCase):
     options.output_format = 'none'
     results = page_runner.Run(Test('RunTest'), ps, expectations, options)
     self.assertEquals(0, len(results.successes))
+    self.assertEquals(0, len(results.failures))
     self.assertEquals(1, len(results.errors))
 
+  def testHandlingOfCrashedTabWithExpectedFailure(self):
+    ps = page_set.PageSet()
+    expectations = test_expectations.TestExpectations()
+    expectations.Fail('chrome://crash')
+    page1 = page_module.Page('chrome://crash', ps)
+    ps.pages.append(page1)
+
+    class Test(page_test.PageTest):
+      def RunTest(self, *args):
+        pass
+
+    options = options_for_unittests.GetCopy()
+    options.output_format = 'none'
+    results = page_runner.Run(Test('RunTest'), ps, expectations, options)
+    self.assertEquals(1, len(results.successes))
+    self.assertEquals(0, len(results.failures))
+    self.assertEquals(0, len(results.errors))
+
   def testDiscardFirstResult(self):
     ps = page_set.PageSet()
     expectations = test_expectations.TestExpectations()