Modified output format to be accepted by Pulse. Updated DEPS to include newer revision from googlecode, which has screenshot name changes.

Review URL: http://codereview.chromium.org/235041

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@27545 0039d316-1c4b-4281-b951-d872f2087c98

3 files changed, 64 insertions, 56 deletions

diff --git a/o3d/tests/selenium/main.py b/o3d/tests/selenium/main.py
index 44075eb..cc19ad7 100644
--- a/o3d/tests/selenium/main.py
+++ b/o3d/tests/selenium/main.py
@@ -384,7 +384,7 @@ class SeleniumSessionBuilder:
     return new_session
 
 
-def TestBrowser(session_builder, browser, test_list):
+def TestBrowser(session_builder, browser, test_list, verbose):
   """Runs Selenium tests for a specific browser.
 
   Args:
@@ -397,7 +397,8 @@ def TestBrowser(session_builder, browser, test_list):
   """
   print "Testing %s..." % browser
   
-  summary_result = test_runner.TestResult(test_runner.StringBuffer(), browser)
+  summary_result = test_runner.TestResult(test_runner.StringBuffer(), browser,
+                                          verbose)
   
   # Fill up the selenium test queue.
   test_queue = Queue.Queue()
@@ -415,12 +416,13 @@ def TestBrowser(session_builder, browser, test_list):
     pdiff_result_queue = Queue.Queue()
     pdiff_worker = test_runner.PDiffTestRunner(pdiff_queue,
                                                pdiff_result_queue,
-                                               browser)
+                                               browser, verbose)
     pdiff_worker.start()
     
   # Start initial selenium test runner.
   worker = test_runner.SeleniumTestRunner(session_builder, browser,
-                                          test_queue, pdiff_queue)
+                                          test_queue, pdiff_queue,
+                                          verbose)
   worker.start()
 
   # Run through all selenium tests.
@@ -434,6 +436,12 @@ def TestBrowser(session_builder, browser, test_list):
       result = worker.Continue()
       result.printAll(sys.stdout)
       summary_result.merge(result)
+    
+    # Sleep here for a brief time.  This thread is polling the worker thread.
+    # We cannot wait for a message from the worker thread because the worker
+    # may hang on a bad test. We also do not want to sleep till the test's
+    # deadline because the test may finish before then.
+    time.sleep(.1)
 
   if FLAGS.screenshots:
     # Finish screenshot comparisons.
@@ -506,16 +514,22 @@ def _GetTestsFromFile(filename, prefix, test_prefix_filter, test_suffixes,
     options = arguments[2:]
 
     # TODO: Add filter based on test_type
+    test_skipped = False
     if test_path.startswith("Test"):
       name = test_path
     else:
       # Need to make a name.
       name = ("Test" + prefix + re.sub("\W", "_", test_path) +
               test_type.capitalize())
-
+      # Only test suffixes for generic tests. That is how it has always worked.
+      if test_suffixes and not MatchesSuffix(name, test_suffixes):
+        test_skipped = True
+    
+    if test_prefix_filter and not name.startswith(test_prefix_filter):
+      test_skipped = True
+      
     # Only execute this test if the current browser is not in the list
     # of skipped browsers.
-    test_skipped = False
     screenshot_count = 0
     for option in options:
       if option.startswith("except"):
@@ -529,10 +543,6 @@ def _GetTestsFromFile(filename, prefix, test_prefix_filter, test_suffixes,
       elif option.startswith("screenshot"):
         screenshot_count += 1
 
-    if (test_prefix_filter and not name.startswith(test_prefix_filter) or
-        test_suffixes and not MatchesSuffix(name, test_suffixes)):
-      test_skipped = True
-
     if not test_skipped:
       # Add a test method with this name if it doesn't exist.
       if not (hasattr(module, name) and callable(getattr(module, name))):
@@ -654,7 +664,7 @@ def main(unused_argv):
       test_list = GetTestsForBrowser(browser, FLAGS.testprefix,
                                      FLAGS.testsuffixes)
 
-      result = TestBrowser(session_builder, browser, test_list)
+      result = TestBrowser(session_builder, browser, test_list, FLAGS.verbose)
 
       if not result.wasSuccessful():
         all_tests_passed = False
diff --git a/o3d/tests/selenium/pdiff_test.py b/o3d/tests/selenium/pdiff_test.py
index fc63c9d..0f8b58c 100644
--- a/o3d/tests/selenium/pdiff_test.py
+++ b/o3d/tests/selenium/pdiff_test.py
@@ -63,34 +63,28 @@ class PDiffTest(unittest.TestCase):
     # Loop over number of screenshots.

     for screenshot_no in range(self.num_screenshots):

       # Find reference image.

+      shotname = self.screenshot_name + str(screenshot_no + 1)

       J = os.path.join

       platform_img_path = J(self.ref_dir,

                             selenium_constants.PLATFORM_SCREENSHOT_DIR,

-                            self.screenshot_name + str(screenshot_no + 1) +

-                            '_reference.png')

+                            shotname + '_reference.png')

       reg_img_path = J(self.ref_dir,

                        selenium_constants.DEFAULT_SCREENSHOT_DIR,

-                       self.screenshot_name + str(screenshot_no + 1) +

-                       '_reference.png')

+                       shotname + '_reference.png')

 

       if os.path.exists(platform_img_path):

         ref_img_path = platform_img_path

       elif os.path.exists(reg_img_path):

         ref_img_path = reg_img_path

       else:

-        self.fail('Reference image for ' + self.screenshot_name + ' not found.'

-                  + '\nNeither file exists %s NOR %s' %

-                  (reg_img_path, platform_img_path))

+        self.fail('Reference image for ' + shotname + ' not found.')

   

       # Find generated image.

-      gen_img_path = J(self.gen_dir, self.screenshot_name +

-                       str(screenshot_no + 1) + '.png')

-      diff_img_path = J(self.gen_dir, 'cmp_' + self.screenshot_name +

-                        str(screenshot_no + 1) + '.png')

+      gen_img_path = J(self.gen_dir, shotname + '.png')

+      diff_img_path = J(self.gen_dir, 'cmp_' + shotname + '.png')

        

       self.assertTrue(os.path.exists(gen_img_path),

-                      'Generated screenshot for ' + self.screenshot_name +

-                      ' not found.\nFile does not exist: %s' % gen_img_path)

+                      'Generated screenshot for ' + shotname + ' not found.\n')

 

       # Run perceptual diff

       arguments = [self.pdiff_path,

@@ -126,14 +120,13 @@ class PDiffTest(unittest.TestCase):
       if pixel_match:

         different_pixels = pixel_match.group(1)

 

-      results += [(gen_img_path, int(different_pixels))]

+      results += [(shotname, int(different_pixels))]

 

     all_tests_passed = True

-    msg = "Pixel Threshold is %s. Failing screenshots:\n" % pixel_threshold

-    for path, pixels in results:

+    msg = "Pixel threshold is %s. Failing screenshots:\n" % pixel_threshold

+    for name, pixels in results:

       if pixels >= pixel_threshold:

         all_tests_passed = False

-        msg += "     %s, differing by %s\n" % (path, str(pixels))

+        msg += "     %s, differing by %s\n" % (name, str(pixels))

 

-    if not all_tests_passed:

-      self.assertTrue(all_tests_passed, msg)

+    self.assertTrue(all_tests_passed, msg)

diff --git a/o3d/tests/selenium/test_runner.py b/o3d/tests/selenium/test_runner.py
index 0ff5789..c739045 100644
--- a/o3d/tests/selenium/test_runner.py
+++ b/o3d/tests/selenium/test_runner.py
@@ -74,10 +74,10 @@ class TestResult(unittest.TestResult):
   """A specialized class that prints formatted text results to a stream.

 

   """

-  separator1 = "=" * 70

-  separator2 = "-" * 70

+  separator1 = "=" * 30

+  separator2 = "-" * 30

 

-  def __init__(self, stream, browser):

+  def __init__(self, stream, browser, verbose):

     unittest.TestResult.__init__(self)

     self.stream = stream

     # Dictionary of start times

@@ -85,6 +85,7 @@ class TestResult(unittest.TestResult):
     # Dictionary of results

     self.results = {}

     self.browser = browser

+    self.verbose = verbose

 

   def getDescription(self, test):

     """Gets description of test."""

@@ -97,10 +98,11 @@ class TestResult(unittest.TestResult):
     # Default testresult if success not called

     self.results[test] = "FAIL"

     unittest.TestResult.startTest(self, test)

-    self.stream.writeln()

-    self.stream.writeln(self.separator2)

-    self.stream.write(self.getDescription(test))

-    self.stream.writeln(" ... ")

+    if self.verbose:

+      self.stream.writeln()

+      self.stream.writeln(self.separator2)

+      self.stream.write(self.getDescription(test))

+      self.stream.writeln(" ... ")

 

   def stopTest(self, test):

     """Called when test is ended."""

@@ -108,6 +110,7 @@ class TestResult(unittest.TestResult):
     result = self.results[test]

     self.stream.writeln("SELENIUMRESULT %s <%s> [%.3fs]: %s"

                         % (test, self.browser, time_taken, result))

+    self.printErrors()

 

   def addSuccess(self, test):

     """Adds success result to TestResult."""

@@ -130,19 +133,21 @@ class TestResult(unittest.TestResult):
     self.testsRun += 1

     self.errors.append("No response from test")

 

-    self.stream.writeln()

-    self.stream.writeln(self.separator2)

-    self.stream.write(self.getDescription(test))

-    self.stream.writeln(" ... ")

-    self.stream.writeln("SELENIUMRESULT %s <%s> [?s]: FAIL (HUNG?)"

+    if self.verbose:

+      self.stream.writeln()

+      self.stream.writeln(self.separator2)

+      self.stream.write(self.getDescription(test))

+      self.stream.writeln(" ... ")

+    self.stream.writeln("SELENIUMRESULT %s <%s> [0s]: FAIL"

                         % (test, self.browser))

-    self.stream.writeln()

+    self.stream.writeln("Test was aborted due to timeout")

 

   def printErrors(self):

     """Prints all errors and failures."""

-    self.stream.writeln()

-    self.printErrorList("ERROR", self.errors)

-    self.printErrorList("FAIL", self.failures)

+    if self.errors:

+      self.printErrorList("ERROR", self.errors)

+    if self.failures:

+      self.printErrorList("FAIL", self.failures)

 

   def printErrorList(self, flavour, errors):

     """Prints a given list of errors."""

@@ -175,14 +180,15 @@ class TestRunnerThread(threading.Thread):
     test: the currently running test.

     browser: selenium_name of browser that will be tested.

   """

-  def __init__(self):

+  def __init__(self, verbose):

     threading.Thread.__init__(self)

     # This thread is a daemon so that the program can exit even if the

     # thread has not finished.

     self.setDaemon(True)

     self.completely_done_event = threading.Event()

-    self.test_copy = None

+    self.test = None

     self.browser = "default_browser"

+    self.verbose = verbose

 

   def IsCompletelyDone(self):

     """Returns true if this test runner is completely done."""

@@ -206,15 +212,14 @@ class TestRunnerThread(threading.Thread):
     self.test = test

 

     stream = StringBuffer()

-    result = TestResult(stream, self.browser)

+    result = TestResult(stream, self.browser, self.verbose)

     startTime = time.time()

     test(result)

     stopTime = time.time()

     timeTaken = stopTime - startTime

-    result.printErrors()

+    if self.verbose:

+      result.printErrors()

     run = result.testsRun

-    stream.writeln("Took %.2fs" % timeTaken)

-    stream.writeln()

     return result

   

 

@@ -229,8 +234,8 @@ class PDiffTestRunner(TestRunnerThread):
     end_testing_event: event that occurs when we are guaranteed no more tests

       will be added to the queue.

   """

-  def __init__(self, pdiff_queue, result_queue, browser):

-    TestRunnerThread.__init__(self)

+  def __init__(self, pdiff_queue, result_queue, browser, verbose):

+    TestRunnerThread.__init__(self, verbose)

     self.pdiff_queue = pdiff_queue

     self.result_queue = result_queue

     self.browser = browser

@@ -278,8 +283,8 @@ class SeleniumTestRunner(TestRunnerThread):
       diff test to the queue when the related selenium test passes.

     deadline: absolute time of when the test should be done.

   """

-  def __init__(self, sel_builder, browser, test_queue, pdiff_queue):

-    TestRunnerThread.__init__(self)

+  def __init__(self, sel_builder, browser, test_queue, pdiff_queue, verbose):

+    TestRunnerThread.__init__(self, verbose)

 

     # Synchronization.

     self.testing_event = threading.Event()