From 234b34431cfc4a69fe36386991291ecc88937d19 Mon Sep 17 00:00:00 2001 From: "kkania@google.com" Date: Tue, 29 Sep 2009 22:22:35 +0000 Subject: Modified output format to be accepted by Pulse. Updated DEPS to include newer revision from googlecode, which has screenshot name changes. Review URL: http://codereview.chromium.org/235041 git-svn-id: svn://svn.chromium.org/chrome/trunk/src@27545 0039d316-1c4b-4281-b951-d872f2087c98 --- o3d/tests/selenium/main.py | 32 ++++++++++++++-------- o3d/tests/selenium/pdiff_test.py | 31 +++++++++------------ o3d/tests/selenium/test_runner.py | 57 +++++++++++++++++++++------------------ 3 files changed, 64 insertions(+), 56 deletions(-) (limited to 'o3d/tests') diff --git a/o3d/tests/selenium/main.py b/o3d/tests/selenium/main.py index 44075eb..cc19ad7 100644 --- a/o3d/tests/selenium/main.py +++ b/o3d/tests/selenium/main.py @@ -384,7 +384,7 @@ class SeleniumSessionBuilder: return new_session -def TestBrowser(session_builder, browser, test_list): +def TestBrowser(session_builder, browser, test_list, verbose): """Runs Selenium tests for a specific browser. Args: @@ -397,7 +397,8 @@ def TestBrowser(session_builder, browser, test_list): """ print "Testing %s..." % browser - summary_result = test_runner.TestResult(test_runner.StringBuffer(), browser) + summary_result = test_runner.TestResult(test_runner.StringBuffer(), browser, + verbose) # Fill up the selenium test queue. test_queue = Queue.Queue() @@ -415,12 +416,13 @@ def TestBrowser(session_builder, browser, test_list): pdiff_result_queue = Queue.Queue() pdiff_worker = test_runner.PDiffTestRunner(pdiff_queue, pdiff_result_queue, - browser) + browser, verbose) pdiff_worker.start() # Start initial selenium test runner. worker = test_runner.SeleniumTestRunner(session_builder, browser, - test_queue, pdiff_queue) + test_queue, pdiff_queue, + verbose) worker.start() # Run through all selenium tests. @@ -434,6 +436,12 @@ def TestBrowser(session_builder, browser, test_list): result = worker.Continue() result.printAll(sys.stdout) summary_result.merge(result) + + # Sleep here for a brief time. This thread is polling the worker thread. + # We cannot wait for a message from the worker thread because the worker + # may hang on a bad test. We also do not want to sleep till the test's + # deadline because the test may finish before then. + time.sleep(.1) if FLAGS.screenshots: # Finish screenshot comparisons. @@ -506,16 +514,22 @@ def _GetTestsFromFile(filename, prefix, test_prefix_filter, test_suffixes, options = arguments[2:] # TODO: Add filter based on test_type + test_skipped = False if test_path.startswith("Test"): name = test_path else: # Need to make a name. name = ("Test" + prefix + re.sub("\W", "_", test_path) + test_type.capitalize()) - + # Only test suffixes for generic tests. That is how it has always worked. + if test_suffixes and not MatchesSuffix(name, test_suffixes): + test_skipped = True + + if test_prefix_filter and not name.startswith(test_prefix_filter): + test_skipped = True + # Only execute this test if the current browser is not in the list # of skipped browsers. - test_skipped = False screenshot_count = 0 for option in options: if option.startswith("except"): @@ -529,10 +543,6 @@ def _GetTestsFromFile(filename, prefix, test_prefix_filter, test_suffixes, elif option.startswith("screenshot"): screenshot_count += 1 - if (test_prefix_filter and not name.startswith(test_prefix_filter) or - test_suffixes and not MatchesSuffix(name, test_suffixes)): - test_skipped = True - if not test_skipped: # Add a test method with this name if it doesn't exist. if not (hasattr(module, name) and callable(getattr(module, name))): @@ -654,7 +664,7 @@ def main(unused_argv): test_list = GetTestsForBrowser(browser, FLAGS.testprefix, FLAGS.testsuffixes) - result = TestBrowser(session_builder, browser, test_list) + result = TestBrowser(session_builder, browser, test_list, FLAGS.verbose) if not result.wasSuccessful(): all_tests_passed = False diff --git a/o3d/tests/selenium/pdiff_test.py b/o3d/tests/selenium/pdiff_test.py index fc63c9d..0f8b58c 100644 --- a/o3d/tests/selenium/pdiff_test.py +++ b/o3d/tests/selenium/pdiff_test.py @@ -63,34 +63,28 @@ class PDiffTest(unittest.TestCase): # Loop over number of screenshots. for screenshot_no in range(self.num_screenshots): # Find reference image. + shotname = self.screenshot_name + str(screenshot_no + 1) J = os.path.join platform_img_path = J(self.ref_dir, selenium_constants.PLATFORM_SCREENSHOT_DIR, - self.screenshot_name + str(screenshot_no + 1) + - '_reference.png') + shotname + '_reference.png') reg_img_path = J(self.ref_dir, selenium_constants.DEFAULT_SCREENSHOT_DIR, - self.screenshot_name + str(screenshot_no + 1) + - '_reference.png') + shotname + '_reference.png') if os.path.exists(platform_img_path): ref_img_path = platform_img_path elif os.path.exists(reg_img_path): ref_img_path = reg_img_path else: - self.fail('Reference image for ' + self.screenshot_name + ' not found.' - + '\nNeither file exists %s NOR %s' % - (reg_img_path, platform_img_path)) + self.fail('Reference image for ' + shotname + ' not found.') # Find generated image. - gen_img_path = J(self.gen_dir, self.screenshot_name + - str(screenshot_no + 1) + '.png') - diff_img_path = J(self.gen_dir, 'cmp_' + self.screenshot_name + - str(screenshot_no + 1) + '.png') + gen_img_path = J(self.gen_dir, shotname + '.png') + diff_img_path = J(self.gen_dir, 'cmp_' + shotname + '.png') self.assertTrue(os.path.exists(gen_img_path), - 'Generated screenshot for ' + self.screenshot_name + - ' not found.\nFile does not exist: %s' % gen_img_path) + 'Generated screenshot for ' + shotname + ' not found.\n') # Run perceptual diff arguments = [self.pdiff_path, @@ -126,14 +120,13 @@ class PDiffTest(unittest.TestCase): if pixel_match: different_pixels = pixel_match.group(1) - results += [(gen_img_path, int(different_pixels))] + results += [(shotname, int(different_pixels))] all_tests_passed = True - msg = "Pixel Threshold is %s. Failing screenshots:\n" % pixel_threshold - for path, pixels in results: + msg = "Pixel threshold is %s. Failing screenshots:\n" % pixel_threshold + for name, pixels in results: if pixels >= pixel_threshold: all_tests_passed = False - msg += " %s, differing by %s\n" % (path, str(pixels)) + msg += " %s, differing by %s\n" % (name, str(pixels)) - if not all_tests_passed: - self.assertTrue(all_tests_passed, msg) + self.assertTrue(all_tests_passed, msg) diff --git a/o3d/tests/selenium/test_runner.py b/o3d/tests/selenium/test_runner.py index 0ff5789..c739045 100644 --- a/o3d/tests/selenium/test_runner.py +++ b/o3d/tests/selenium/test_runner.py @@ -74,10 +74,10 @@ class TestResult(unittest.TestResult): """A specialized class that prints formatted text results to a stream. """ - separator1 = "=" * 70 - separator2 = "-" * 70 + separator1 = "=" * 30 + separator2 = "-" * 30 - def __init__(self, stream, browser): + def __init__(self, stream, browser, verbose): unittest.TestResult.__init__(self) self.stream = stream # Dictionary of start times @@ -85,6 +85,7 @@ class TestResult(unittest.TestResult): # Dictionary of results self.results = {} self.browser = browser + self.verbose = verbose def getDescription(self, test): """Gets description of test.""" @@ -97,10 +98,11 @@ class TestResult(unittest.TestResult): # Default testresult if success not called self.results[test] = "FAIL" unittest.TestResult.startTest(self, test) - self.stream.writeln() - self.stream.writeln(self.separator2) - self.stream.write(self.getDescription(test)) - self.stream.writeln(" ... ") + if self.verbose: + self.stream.writeln() + self.stream.writeln(self.separator2) + self.stream.write(self.getDescription(test)) + self.stream.writeln(" ... ") def stopTest(self, test): """Called when test is ended.""" @@ -108,6 +110,7 @@ class TestResult(unittest.TestResult): result = self.results[test] self.stream.writeln("SELENIUMRESULT %s <%s> [%.3fs]: %s" % (test, self.browser, time_taken, result)) + self.printErrors() def addSuccess(self, test): """Adds success result to TestResult.""" @@ -130,19 +133,21 @@ class TestResult(unittest.TestResult): self.testsRun += 1 self.errors.append("No response from test") - self.stream.writeln() - self.stream.writeln(self.separator2) - self.stream.write(self.getDescription(test)) - self.stream.writeln(" ... ") - self.stream.writeln("SELENIUMRESULT %s <%s> [?s]: FAIL (HUNG?)" + if self.verbose: + self.stream.writeln() + self.stream.writeln(self.separator2) + self.stream.write(self.getDescription(test)) + self.stream.writeln(" ... ") + self.stream.writeln("SELENIUMRESULT %s <%s> [0s]: FAIL" % (test, self.browser)) - self.stream.writeln() + self.stream.writeln("Test was aborted due to timeout") def printErrors(self): """Prints all errors and failures.""" - self.stream.writeln() - self.printErrorList("ERROR", self.errors) - self.printErrorList("FAIL", self.failures) + if self.errors: + self.printErrorList("ERROR", self.errors) + if self.failures: + self.printErrorList("FAIL", self.failures) def printErrorList(self, flavour, errors): """Prints a given list of errors.""" @@ -175,14 +180,15 @@ class TestRunnerThread(threading.Thread): test: the currently running test. browser: selenium_name of browser that will be tested. """ - def __init__(self): + def __init__(self, verbose): threading.Thread.__init__(self) # This thread is a daemon so that the program can exit even if the # thread has not finished. self.setDaemon(True) self.completely_done_event = threading.Event() - self.test_copy = None + self.test = None self.browser = "default_browser" + self.verbose = verbose def IsCompletelyDone(self): """Returns true if this test runner is completely done.""" @@ -206,15 +212,14 @@ class TestRunnerThread(threading.Thread): self.test = test stream = StringBuffer() - result = TestResult(stream, self.browser) + result = TestResult(stream, self.browser, self.verbose) startTime = time.time() test(result) stopTime = time.time() timeTaken = stopTime - startTime - result.printErrors() + if self.verbose: + result.printErrors() run = result.testsRun - stream.writeln("Took %.2fs" % timeTaken) - stream.writeln() return result @@ -229,8 +234,8 @@ class PDiffTestRunner(TestRunnerThread): end_testing_event: event that occurs when we are guaranteed no more tests will be added to the queue. """ - def __init__(self, pdiff_queue, result_queue, browser): - TestRunnerThread.__init__(self) + def __init__(self, pdiff_queue, result_queue, browser, verbose): + TestRunnerThread.__init__(self, verbose) self.pdiff_queue = pdiff_queue self.result_queue = result_queue self.browser = browser @@ -278,8 +283,8 @@ class SeleniumTestRunner(TestRunnerThread): diff test to the queue when the related selenium test passes. deadline: absolute time of when the test should be done. """ - def __init__(self, sel_builder, browser, test_queue, pdiff_queue): - TestRunnerThread.__init__(self) + def __init__(self, sel_builder, browser, test_queue, pdiff_queue, verbose): + TestRunnerThread.__init__(self, verbose) # Synchronization. self.testing_event = threading.Event() -- cgit v1.1