diff options
| author | bulach@chromium.org <bulach@chromium.org@0039d316-1c4b-4281-b951-d872f2087c98> | 2012-07-12 11:35:03 +0000 |
|---|---|---|
| committer | bulach@chromium.org <bulach@chromium.org@0039d316-1c4b-4281-b951-d872f2087c98> | 2012-07-12 11:35:03 +0000 |
| commit | 619662cb8e2ab7d355a9fd516bb972f4bcdb6ac1 (patch) | |
| tree | 20446bd457e157d2b346c0a7a54a45a37fd153b0 /build | |
| parent | b1721cc7e49f7b6c850044cfe647d4f28b181e20 (diff) | |
| download | chromium_src-619662cb8e2ab7d355a9fd516bb972f4bcdb6ac1.zip chromium_src-619662cb8e2ab7d355a9fd516bb972f4bcdb6ac1.tar.gz chromium_src-619662cb8e2ab7d355a9fd516bb972f4bcdb6ac1.tar.bz2 | |
Android: a few minor improvements for the test runner.
- We had a few options with default=True, which made them not really
optional. Instead, use an option with action='store_true' which is False by
default.
- Uses buildbot_report.py rather than duplicate the functionality.
- Removes default params from SingleTestRunner to make it more explicit.
BUG=
TEST=existing android tests.
Review URL: https://chromiumcodereview.appspot.com/10689159
git-svn-id: svn://svn.chromium.org/chrome/trunk/src@146318 0039d316-1c4b-4281-b951-d872f2087c98
Diffstat (limited to 'build')
| -rwxr-xr-x | build/android/buildbot_functions.sh | 2 | ||||
| -rw-r--r-- | build/android/pylib/buildbot_report.py | 9 | ||||
| -rw-r--r-- | build/android/pylib/single_test_runner.py | 3 | ||||
| -rwxr-xr-x | build/android/run_tests.py | 158 |
4 files changed, 86 insertions, 86 deletions
diff --git a/build/android/buildbot_functions.sh b/build/android/buildbot_functions.sh index dcea5f5..0e3320c 100755 --- a/build/android/buildbot_functions.sh +++ b/build/android/buildbot_functions.sh @@ -247,7 +247,7 @@ function bb_run_tests_emulator { # Run tests on an actual device. (Better have one plugged in!) function bb_run_tests { echo "@@@BUILD_STEP Run Tests on actual hardware@@@" - build/android/run_tests.py --xvfb --verbose --apk=True + build/android/run_tests.py --xvfb --verbose } # Zip and archive a build. diff --git a/build/android/pylib/buildbot_report.py b/build/android/pylib/buildbot_report.py index fdb5203..1e549de 100644 --- a/build/android/pylib/buildbot_report.py +++ b/build/android/pylib/buildbot_report.py @@ -26,3 +26,12 @@ def PrintMsg(msg): def PrintError(): """Marks the current step as failed.""" print '@@@STEP_FAILURE@@@' + + +def PrintWarning(): + """Marks the current step with a warning.""" + print '@@@STEP_WARNINGS@@@' + + +def PrintNamedStep(step): + print '@@@BUILD_STEP %s@@@' % step diff --git a/build/android/pylib/single_test_runner.py b/build/android/pylib/single_test_runner.py index f465517..f1ee76f 100644 --- a/build/android/pylib/single_test_runner.py +++ b/build/android/pylib/single_test_runner.py @@ -35,8 +35,7 @@ class SingleTestRunner(BaseTestRunner): def __init__(self, device, test_suite, gtest_filter, test_arguments, timeout, rebaseline, performance_test, cleanup_test_files, tool_name, - shard_index, dump_debug_info=False, - fast_and_loose=False): + shard_index, dump_debug_info, fast_and_loose): BaseTestRunner.__init__(self, device, tool_name, shard_index) self._running_on_emulator = self.device.startswith('emulator') self._gtest_filter = gtest_filter diff --git a/build/android/run_tests.py b/build/android/run_tests.py index 44c4b09..b98b41d 100755 --- a/build/android/run_tests.py +++ b/build/android/run_tests.py @@ -51,12 +51,14 @@ failed on device. import fnmatch import logging import os +import signal import subprocess import sys import time from pylib import android_commands from pylib.base_test_sharder import BaseTestSharder +from pylib import buildbot_report from pylib import constants from pylib import debug_info import emulator @@ -66,6 +68,7 @@ from pylib import test_options_parser from pylib.single_test_runner import SingleTestRunner from pylib.test_result import BaseTestResult, TestResults + _TEST_SUITES = ['base_unittests', 'content_unittests', 'gpu_unittests', @@ -76,23 +79,25 @@ _TEST_SUITES = ['base_unittests', 'ui_unittests', ] -def FullyQualifiedTestSuites(apk, test_suites): + +def FullyQualifiedTestSuites(exe, test_suites): """Return a fully qualified list that represents all known suites. Args: - apk: if True, use the apk-based test runner - test_suites: the source test suites to process""" - # If not specified, assume the test suites are in out/Release + exe: if True, use the executable-based test runner. + test_suites: the source test suites to process. + """ + # Assume the test suites are in out/Release. test_suite_dir = os.path.abspath(os.path.join(constants.CHROME_DIR, 'out', 'Release')) - if apk: + if exe: + suites = [os.path.join(test_suite_dir, t) for t in _TEST_SUITES] + else: # out/Release/$SUITE_apk/$SUITE-debug.apk suites = [os.path.join(test_suite_dir, t + '_apk', t + '-debug.apk') for t in test_suites] - else: - suites = [os.path.join(test_suite_dir, t) for t in _TEST_SUITES] return suites @@ -110,10 +115,11 @@ class TimeProfile(object): """Stop profiling and dump a log.""" if self._starttime: stoptime = time.time() - logging.info('%fsec to perform %s' % - (stoptime - self._starttime, self._description)) + logging.info('%fsec to perform %s', + stoptime - self._starttime, self._description) self._starttime = None + class Xvfb(object): """Class to start and stop Xvfb if relevant. Nop if not Linux.""" @@ -131,18 +137,18 @@ class Xvfb(object): """ if not self._IsLinux(): return - proc = subprocess.Popen(["Xvfb", ":9", "-screen", "0", "1024x768x24", - "-ac"], + proc = subprocess.Popen(['Xvfb', ':9', '-screen', '0', '1024x768x24', + '-ac'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) self._pid = proc.pid if not self._pid: raise Exception('Could not start Xvfb') - os.environ['DISPLAY'] = ":9" + os.environ['DISPLAY'] = ':9' # Now confirm, giving a chance for it to start if needed. - for test in range(10): + for _ in range(10): proc = subprocess.Popen('xdpyinfo >/dev/null', shell=True) - pid, retcode = os.waitpid(proc.pid, 0) + _, retcode = os.waitpid(proc.pid, 0) if retcode == 0: break time.sleep(0.25) @@ -159,24 +165,23 @@ class Xvfb(object): del os.environ['DISPLAY'] self._pid = 0 + def PrintAnnotationForTestResults(test_results): if test_results.timed_out: - print '@@@STEP_WARNINGS@@@' - elif test_results.failed: - print '@@@STEP_FAILURE@@@' - elif test_results.crashed: - print '@@@STEP_FAILURE@@@' - elif test_results.overall_fail: - print '@@@STEP_FAILURE@@@' + buildbot_report.PrintWarning() + elif test_results.failed or test_results.crashed or test_results.overall_fail: + buildbot_report.PrintError() else: print 'Step success!' # No annotation needed -def RunTests(device, test_suite, gtest_filter, test_arguments, rebaseline, + +def RunTests(exe, device, test_suite, gtest_filter, test_arguments, rebaseline, timeout, performance_test, cleanup_test_files, tool, - log_dump_name, apk, annotate=False): + log_dump_name, fast_and_loose): """Runs the tests. Args: + exe: boolean to state if we are using the exe based test runner device: Device to run the tests. test_suite: A specific test suite to run, empty to run all. gtest_filter: A gtest_filter flag. @@ -187,47 +192,40 @@ def RunTests(device, test_suite, gtest_filter, test_arguments, rebaseline, cleanup_test_files: Whether or not to cleanup test files on device. tool: Name of the Valgrind tool. log_dump_name: Name of log dump file. - apk: boolean to state if we are using the apk based test runner - annotate: should we print buildbot-style annotations? + fast_and_loose: if set, skip copying data files. Returns: A TestResults object. """ results = [] - global _TEST_SUITES if test_suite: - global _TEST_SUITES - - # If not specified, assume the test suites are in out/Release - test_suite_dir = os.path.abspath(os.path.join(constants.CHROME_DIR, - 'out', 'Release')) - if (not os.path.exists(test_suite)): - logging.critical('Unrecognized test suite %s, supported: %s' % - (test_suite, _TEST_SUITES)) + if not os.path.exists(test_suite): + logging.critical('Unrecognized test suite %s, supported: %s', + test_suite, _TEST_SUITES) if test_suite in _TEST_SUITES: logging.critical('(Remember to include the path: out/Release/%s)', test_suite) test_suite_basename = os.path.basename(test_suite) if test_suite_basename in _TEST_SUITES: - logging.critical('Try "make -j15 %s"' % test_suite_basename) + logging.critical('Try "make -j15 %s"', test_suite_basename) else: - logging.critical('Unrecognized test suite, supported: %s' % + logging.critical('Unrecognized test suite, supported: %s', _TEST_SUITES) return TestResults.FromRun([], [BaseTestResult(test_suite, '')], - False, False) + False, False) fully_qualified_test_suites = [test_suite] else: - fully_qualified_test_suites = FullyQualifiedTestSuites(apk, _TEST_SUITES) + fully_qualified_test_suites = FullyQualifiedTestSuites(exe, _TEST_SUITES) debug_info_list = [] print 'Known suites: ' + str(_TEST_SUITES) print 'Running these: ' + str(fully_qualified_test_suites) for t in fully_qualified_test_suites: - if annotate: - print '@@@BUILD_STEP Test suite %s@@@' % os.path.basename(t) + buildbot_report.PrintNamedStep('Test suite %s' % os.path.basename(t)) test = SingleTestRunner(device, t, gtest_filter, test_arguments, timeout, rebaseline, performance_test, - cleanup_test_files, tool, 0, not not log_dump_name) + cleanup_test_files, tool, 0, not not log_dump_name, + fast_and_loose) test.Run() results += [test.test_results] @@ -238,12 +236,10 @@ def RunTests(device, test_suite, gtest_filter, test_arguments, rebaseline, test.test_results.LogFull('Unit test', os.path.basename(t)) # Zip all debug info outputs into a file named by log_dump_name. debug_info.GTestDebugInfo.ZipAndCleanResults( - os.path.join(constants.CHROME_DIR, 'out', 'Release', - 'debug_info_dumps'), + os.path.join(constants.CHROME_DIR, 'out', 'Release', 'debug_info_dumps'), log_dump_name, [d for d in debug_info_list if d]) - if annotate: - PrintAnnotationForTestResults(test.test_results) + PrintAnnotationForTestResults(test.test_results) return TestResults.FromTestResults(results) @@ -253,7 +249,7 @@ class TestSharder(BaseTestSharder): def __init__(self, attached_devices, test_suite, gtest_filter, test_arguments, timeout, rebaseline, performance_test, - cleanup_test_files, tool, annotate): + cleanup_test_files, tool, log_dump_name, fast_and_loose): BaseTestSharder.__init__(self, attached_devices) self.test_suite = test_suite self.test_suite_basename = os.path.basename(test_suite) @@ -264,10 +260,12 @@ class TestSharder(BaseTestSharder): self.performance_test = performance_test self.cleanup_test_files = cleanup_test_files self.tool = tool - self.annotate = annotate + self.log_dump_name = log_dump_name + self.fast_and_loose = fast_and_loose test = SingleTestRunner(self.attached_devices[0], test_suite, gtest_filter, test_arguments, timeout, rebaseline, - performance_test, cleanup_test_files, tool, 0) + performance_test, cleanup_test_files, tool, 0, + not not self.log_dump_name, fast_and_loose) # The executable/apk needs to be copied before we can call GetAllTests. test.test_package.StripAndCopyExecutable() all_tests = test.test_package.GetAllTests() @@ -297,18 +295,17 @@ class TestSharder(BaseTestSharder): return SingleTestRunner(device, self.test_suite, test_filter, self.test_arguments, self.timeout, self.rebaseline, self.performance_test, - self.cleanup_test_files, self.tool, index) + self.cleanup_test_files, self.tool, index, + not not self.log_dump_name, self.fast_and_loose) def OnTestsCompleted(self, test_runners, test_results): """Notifies that we completed the tests.""" test_results.LogFull('Unit test', os.path.basename(self.test_suite)) - if self.annotate: - PrintAnnotationForTestResults(test_results) + PrintAnnotationForTestResults(test_results) if test_results.failed and self.rebaseline: test_runners[0].UpdateFilter(test_results.failed) - def _RunATestSuite(options): """Run a single test suite. @@ -334,7 +331,7 @@ def _RunATestSuite(options): buildbot_emulators.append(buildbot_emulator) attached_devices.append(buildbot_emulator.device) # Wait for all emulators to boot completed. - map(lambda buildbot_emulator:buildbot_emulator.ConfirmLaunch(True), + map(lambda buildbot_emulator: buildbot_emulator.ConfirmLaunch(True), buildbot_emulators) elif options.test_device: attached_devices = [options.test_device] @@ -343,8 +340,7 @@ def _RunATestSuite(options): if not attached_devices: logging.critical('A device must be attached and online.') - if options.annotate: - print '@@@STEP_FAILURE@@@' + buildbot_report.PrintError() return 1 # Reset the test port allocation. It's important to do it before starting @@ -359,17 +355,16 @@ def _RunATestSuite(options): options.timeout, options.rebaseline, options.performance_test, options.cleanup_test_files, options.tool, - options.annotate) + options.log_dump, options.fast_and_loose) test_results = sharder.RunShardedTests() else: - test_results = RunTests(attached_devices[0], options.test_suite, + test_results = RunTests(options.exe, attached_devices[0], + options.test_suite, options.gtest_filter, options.test_arguments, options.rebaseline, options.timeout, options.performance_test, options.cleanup_test_files, options.tool, - options.log_dump, - options.apk, - annotate=options.annotate) + options.log_dump, options.fast_and_loose) for buildbot_emulator in buildbot_emulators: buildbot_emulator.Shutdown() @@ -380,7 +375,7 @@ def _RunATestSuite(options): if test_results.timed_out and options.repeat: logging.critical('Timed out; repeating in fast_and_loose mode.') options.fast_and_loose = True - options.repeat = options.repeat - 1 + options.repeat -= 1 logging.critical('Repeats left: ' + str(options.repeat)) return _RunATestSuite(options) return len(test_results.failed) @@ -407,10 +402,10 @@ def Dispatch(options): xvfb.Start() if options.test_suite: - all_test_suites = FullyQualifiedTestSuites(options.apk, + all_test_suites = FullyQualifiedTestSuites(options.exe, [options.test_suite]) else: - all_test_suites = FullyQualifiedTestSuites(options.apk, + all_test_suites = FullyQualifiedTestSuites(options.exe, _TEST_SUITES) failures = 0 for suite in all_test_suites: @@ -423,16 +418,15 @@ def Dispatch(options): def ListTestSuites(): - """Display a list of available test suites - """ + """Display a list of available test suites.""" print 'Available test suites are:' for test_suite in _TEST_SUITES: print test_suite def main(argv): - option_parser = test_options_parser.CreateTestRunnerOptionParser(None, - default_timeout=0) + option_parser = test_options_parser.CreateTestRunnerOptionParser( + None, default_timeout=0) option_parser.add_option('-s', '--suite', dest='test_suite', help='Executable name of the test suite to run ' '(use -s help to list them)') @@ -440,16 +434,14 @@ def main(argv): help='Target device the test suite to run ') option_parser.add_option('-r', dest='rebaseline', help='Rebaseline and update *testsuite_disabled', - action='store_true', - default=False) + action='store_true') option_parser.add_option('-f', '--gtest_filter', dest='gtest_filter', help='gtest filter') option_parser.add_option('-a', '--test_arguments', dest='test_arguments', help='Additional arguments to pass to the test') option_parser.add_option('-p', dest='performance_test', help='Indicator of performance test', - action='store_true', - default=False) + action='store_true') option_parser.add_option('-L', dest='log_dump', help='file name of log dump, which will be put in' 'subfolder debug_info_dumps under the same directory' @@ -459,10 +451,10 @@ def main(argv): type='int', default=0) option_parser.add_option('-x', '--xvfb', dest='use_xvfb', - action='store_true', default=False, + action='store_true', help='Use Xvfb around tests (ignored if not Linux)') option_parser.add_option('--fast', '--fast_and_loose', dest='fast_and_loose', - action='store_true', default=False, + action='store_true', help='Go faster (but be less stable), ' 'for quick testing. Example: when tracking down ' 'tests that hang to add to the disabled list, ' @@ -472,11 +464,12 @@ def main(argv): option_parser.add_option('--repeat', dest='repeat', type='int', default=2, help='Repeat count on test timeout') - option_parser.add_option('--annotate', default=True, - help='Print buildbot-style annotate messages ' - 'for each test suite. Default=True') - option_parser.add_option('--apk', default=True, - help='Use the apk test runner by default') + option_parser.add_option('--exit_code', action='store_true', + help='If set, the exit code will be total number ' + 'of failures.') + option_parser.add_option('--exe', action='store_true', + help='If set, use the exe test runner instead of ' + 'the APK.') options, args = option_parser.parse_args(argv) if len(args) > 1: print 'Unknown argument:', args[1:] @@ -485,17 +478,16 @@ def main(argv): run_tests_helper.SetLogLevel(options.verbose_count) failed_tests_count = Dispatch(options) - # If we're printing annotations then failures of individual test suites are - # communicated by printing a STEP_FAILURE message. + # Failures of individual test suites are communicated by printing a + # STEP_FAILURE message. # Returning a success exit status also prevents the buildbot from incorrectly # marking the last suite as failed if there were failures in other suites in # the batch (this happens because the exit status is a sum of all failures # from all suites, but the buildbot associates the exit status only with the # most recent step). - if options.annotate: - return 0 - else: + if options.exit_code: return failed_tests_count + return 0 if __name__ == '__main__': |