diff options
author | jbudorick <jbudorick@chromium.org> | 2015-09-18 11:57:39 -0700 |
---|---|---|
committer | Commit bot <commit-bot@chromium.org> | 2015-09-18 18:58:25 +0000 |
commit | 605b8939b63975fc3cf960bec96d58a3578543a6 (patch) | |
tree | 50376ab4913b6061f050cef74f040036a8c07890 /build | |
parent | 6d2a30316defffdd77c473547063316ebb9c279b (diff) | |
download | chromium_src-605b8939b63975fc3cf960bec96d58a3578543a6.zip chromium_src-605b8939b63975fc3cf960bec96d58a3578543a6.tar.gz chromium_src-605b8939b63975fc3cf960bec96d58a3578543a6.tar.bz2 |
Revert of [Android] Remove old gtest test_runner code. (patchset #2 id:20001 of https://codereview.chromium.org/1353603003/ )
Reason for revert:
need to revert predecessor
Original issue's description:
> [Android] Remove old gtest test_runner code.
>
> BUG=428729
>
> Committed: https://crrev.com/9227d865c8b4eb0c10086efe4213c9f84457ee03
> Cr-Commit-Position: refs/heads/master@{#349694}
TBR=mikecase@chromium.org,rnephew@chromium.org,dgrogan@chromium.org
NOPRESUBMIT=true
NOTREECHECKS=true
NOTRY=true
BUG=428729
Review URL: https://codereview.chromium.org/1354213004
Cr-Commit-Position: refs/heads/master@{#349728}
Diffstat (limited to 'build')
-rw-r--r-- | build/all.gyp | 4 | ||||
-rwxr-xr-x | build/android/buildbot/bb_device_steps.py | 48 | ||||
-rw-r--r-- | build/android/pylib/gtest/gtest_config.py | 54 | ||||
-rw-r--r-- | build/android/pylib/gtest/setup.py | 229 | ||||
-rw-r--r-- | build/android/pylib/gtest/test_options.py | 19 | ||||
-rw-r--r-- | build/android/pylib/gtest/test_package.py | 80 | ||||
-rw-r--r-- | build/android/pylib/gtest/test_package_apk.py | 171 | ||||
-rw-r--r-- | build/android/pylib/gtest/test_package_exe.py | 163 | ||||
-rw-r--r-- | build/android/pylib/gtest/test_runner.py | 218 | ||||
-rwxr-xr-x | build/android/test_runner.py | 48 |
10 files changed, 989 insertions, 45 deletions
diff --git a/build/all.gyp b/build/all.gyp index 9061f29..953ec13d 100644 --- a/build/all.gyp +++ b/build/all.gyp @@ -783,7 +783,9 @@ 'targets': [ { # The current list of tests for android. This is temporary - # until the full set supported. + # until the full set supported. If adding a new test here, + # please also add it to build/android/pylib/gtest/gtest_config.py, + # else the test is not run. # # WARNING: # Do not add targets here without communicating the implications diff --git a/build/android/buildbot/bb_device_steps.py b/build/android/buildbot/bb_device_steps.py index f4c84a8..e4d2998 100755 --- a/build/android/buildbot/bb_device_steps.py +++ b/build/android/buildbot/bb_device_steps.py @@ -20,6 +20,7 @@ sys.path.append(os.path.join(os.path.dirname(__file__), '..')) import provision_devices from devil.android import device_utils from pylib import constants +from pylib.gtest import gtest_config CHROME_SRC_DIR = bb_utils.CHROME_SRC DIR_BUILD_ROOT = os.path.dirname(CHROME_SRC_DIR) @@ -31,46 +32,6 @@ LOGCAT_DIR = os.path.join(bb_utils.CHROME_OUT_DIR, 'logcat') GS_URL = 'https://storage.googleapis.com' GS_AUTH_URL = 'https://storage.cloud.google.com' -# gtest suite groups. -ASAN_EXCLUDED_GTEST_SUITES = [ - 'breakpad_unittests', - 'sandbox_linux_unittests' -] - -EXPERIMENTAL_GTEST_SUITES = [ - 'components_browsertests', - 'content_gl_tests', - 'heap_profiler_unittests', - 'devtools_bridge_tests', -] - -STABLE_GTEST_SUITES = [ - 'android_webview_unittests', - 'base_unittests', - 'breakpad_unittests', - 'cc_unittests', - 'components_unittests', - 'content_browsertests', - 'content_unittests', - 'events_unittests', - 'gl_tests', - 'gl_unittests', - 'gpu_unittests', - 'ipc_tests', - 'media_unittests', - 'midi_unittests', - 'net_unittests', - 'sandbox_linux_unittests', - 'skia_unittests', - 'sql_unittests', - 'sync_unit_tests', - 'ui_android_unittests', - 'ui_base_unittests', - 'ui_touch_selection_unittests', - 'unit_tests', - 'webkit_unit_tests', -] - # Describes an instrumation test suite: # test: Name of test we're running. # apk: apk to be installed. @@ -88,6 +49,7 @@ I_TEST = collections.namedtuple('InstrumentationTest', [ def SrcPath(*path): return os.path.join(CHROME_SRC_DIR, *path) + def I(name, apk, apk_package, test_apk, test_data, isolate_file_path=None, host_driven_root=None, annotation=None, exclude_annotation=None, extra_flags=None): @@ -517,10 +479,10 @@ def GetDeviceSetupStepCmds(): def RunUnitTests(options): - suites = STABLE_GTEST_SUITES + suites = gtest_config.STABLE_TEST_SUITES if options.asan: suites = [s for s in suites - if s not in ASAN_EXCLUDED_GTEST_SUITES] + if s not in gtest_config.ASAN_EXCLUDED_TEST_SUITES] RunTestSuites(options, suites) @@ -710,7 +672,7 @@ def MainTestWrapper(options): shutil.rmtree(coverage_html, ignore_errors=True) if options.experimental: - RunTestSuites(options, EXPERIMENTAL_GTEST_SUITES) + RunTestSuites(options, gtest_config.EXPERIMENTAL_TEST_SUITES) finally: # Run all post test steps diff --git a/build/android/pylib/gtest/gtest_config.py b/build/android/pylib/gtest/gtest_config.py new file mode 100644 index 0000000..76e0f50 --- /dev/null +++ b/build/android/pylib/gtest/gtest_config.py @@ -0,0 +1,54 @@ +# Copyright (c) 2013 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Configuration file for android gtest suites.""" + +# Add new suites here before upgrading them to the stable list below. +EXPERIMENTAL_TEST_SUITES = [ + 'components_browsertests', + 'content_gl_tests', + 'heap_profiler_unittests', + 'devtools_bridge_tests', +] + +TELEMETRY_EXPERIMENTAL_TEST_SUITES = [ + 'telemetry_unittests', +] + +# Do not modify this list without approval of an android owner. +# This list determines which suites are run by default, both for local +# testing and on android trybots running on commit-queue. +STABLE_TEST_SUITES = [ + 'android_webview_unittests', + 'base_unittests', + 'breakpad_unittests', + 'cc_unittests', + 'components_unittests', + 'content_browsertests', + 'content_unittests', + 'events_unittests', + 'gl_tests', + 'gl_unittests', + 'gpu_unittests', + 'ipc_tests', + 'media_unittests', + 'midi_unittests', + 'net_unittests', + 'sandbox_linux_unittests', + 'skia_unittests', + 'sql_unittests', + 'sync_unit_tests', + 'ui_android_unittests', + 'ui_base_unittests', + 'ui_touch_selection_unittests', + 'unit_tests', + 'webkit_unit_tests', +] + +# Tests fail in component=shared_library build, which is required for ASan. +# http://crbug.com/344868 +ASAN_EXCLUDED_TEST_SUITES = [ + 'breakpad_unittests', + 'sandbox_linux_unittests' +] diff --git a/build/android/pylib/gtest/setup.py b/build/android/pylib/gtest/setup.py new file mode 100644 index 0000000..1fca46c --- /dev/null +++ b/build/android/pylib/gtest/setup.py @@ -0,0 +1,229 @@ +# Copyright 2013 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Generates test runner factory and tests for GTests.""" +# pylint: disable=W0212 + +import logging +import os +import sys + +from devil.android import device_utils +from pylib import constants +from pylib.base import base_setup +from pylib.base import base_test_result +from pylib.base import test_dispatcher +from pylib.gtest import gtest_test_instance +from pylib.gtest import test_package_apk +from pylib.gtest import test_package_exe +from pylib.gtest import test_runner + +sys.path.insert(0, + os.path.join(constants.DIR_SOURCE_ROOT, 'build', 'util', 'lib', + 'common')) +import unittest_util # pylint: disable=F0401 + + +ISOLATE_FILE_PATHS = gtest_test_instance._DEFAULT_ISOLATE_FILE_PATHS + + +# Used for filtering large data deps at a finer grain than what's allowed in +# isolate files since pushing deps to devices is expensive. +# Wildcards are allowed. +DEPS_EXCLUSION_LIST = [ + 'chrome/test/data/extensions/api_test', + 'chrome/test/data/extensions/secure_shell', + 'chrome/test/data/firefox*', + 'chrome/test/data/gpu', + 'chrome/test/data/image_decoding', + 'chrome/test/data/import', + 'chrome/test/data/page_cycler', + 'chrome/test/data/perf', + 'chrome/test/data/pyauto_private', + 'chrome/test/data/safari_import', + 'chrome/test/data/scroll', + 'chrome/test/data/third_party', + 'third_party/hunspell_dictionaries/*.dic', + # crbug.com/258690 + 'webkit/data/bmp_decoder', + 'webkit/data/ico_decoder', +] + + +def _GetDisabledTestsFilterFromFile(suite_name): + """Returns a gtest filter based on the *_disabled file. + + Args: + suite_name: Name of the test suite (e.g. base_unittests). + + Returns: + A gtest filter which excludes disabled tests. + Example: '*-StackTrace.*:StringPrintfTest.StringPrintfMisc' + """ + filter_file_path = os.path.join( + os.path.abspath(os.path.dirname(__file__)), + 'filter', '%s_disabled' % suite_name) + + if not filter_file_path or not os.path.exists(filter_file_path): + logging.info('No filter file found at %s', filter_file_path) + return '*' + + filters = [x for x in [x.strip() for x in file(filter_file_path).readlines()] + if x and x[0] != '#'] + disabled_filter = '*-%s' % ':'.join(filters) + logging.info('Applying filter "%s" obtained from %s', + disabled_filter, filter_file_path) + return disabled_filter + + +def _GetTests(test_options, test_package, devices): + """Get a list of tests. + + Args: + test_options: A GTestOptions object. + test_package: A TestPackageApk object. + devices: A list of attached devices. + + Returns: + A list of all the tests in the test suite. + """ + class TestListResult(base_test_result.BaseTestResult): + def __init__(self): + super(TestListResult, self).__init__( + 'gtest_list_tests', base_test_result.ResultType.PASS) + self.test_list = [] + + def TestListerRunnerFactory(device, _shard_index): + class TestListerRunner(test_runner.TestRunner): + def RunTest(self, _test): + result = TestListResult() + self.test_package.Install(self.device) + result.test_list = self.test_package.GetAllTests(self.device) + results = base_test_result.TestRunResults() + results.AddResult(result) + return results, None + return TestListerRunner(test_options, device, test_package) + + results, _ = test_dispatcher.RunTests( + ['gtest_list_tests'], TestListerRunnerFactory, devices) + tests = [] + for r in results.GetAll(): + tests.extend(r.test_list) + return tests + + +def _FilterTestsUsingPrefixes(all_tests, pre=False, manual=False): + """Removes tests with disabled prefixes. + + Args: + all_tests: List of tests to filter. + pre: If True, include tests with PRE_ prefix. + manual: If True, include tests with MANUAL_ prefix. + + Returns: + List of tests remaining. + """ + filtered_tests = [] + filter_prefixes = ['DISABLED_', 'FLAKY_', 'FAILS_'] + + if not pre: + filter_prefixes.append('PRE_') + + if not manual: + filter_prefixes.append('MANUAL_') + + for t in all_tests: + test_case, test = t.split('.', 1) + if not any([test_case.startswith(prefix) or test.startswith(prefix) for + prefix in filter_prefixes]): + filtered_tests.append(t) + return filtered_tests + + +def _FilterDisabledTests(tests, suite_name, has_gtest_filter): + """Removes disabled tests from |tests|. + + Applies the following filters in order: + 1. Remove tests with disabled prefixes. + 2. Remove tests specified in the *_disabled files in the 'filter' dir + + Args: + tests: List of tests. + suite_name: Name of the test suite (e.g. base_unittests). + has_gtest_filter: Whether a gtest_filter is provided. + + Returns: + List of tests remaining. + """ + tests = _FilterTestsUsingPrefixes( + tests, has_gtest_filter, has_gtest_filter) + tests = unittest_util.FilterTestNames( + tests, _GetDisabledTestsFilterFromFile(suite_name)) + + return tests + + +def Setup(test_options, devices): + """Create the test runner factory and tests. + + Args: + test_options: A GTestOptions object. + devices: A list of attached devices. + + Returns: + A tuple of (TestRunnerFactory, tests). + """ + test_package = test_package_apk.TestPackageApk(test_options.suite_name) + if not os.path.exists(test_package.suite_path): + exe_test_package = test_package_exe.TestPackageExecutable( + test_options.suite_name) + if not os.path.exists(exe_test_package.suite_path): + raise Exception( + 'Did not find %s target. Ensure it has been built.\n' + '(not found at %s or %s)' + % (test_options.suite_name, + test_package.suite_path, + exe_test_package.suite_path)) + test_package = exe_test_package + logging.warning('Found target %s', test_package.suite_path) + + i = base_setup.GenerateDepsDirUsingIsolate(test_options.suite_name, + test_options.isolate_file_path, + ISOLATE_FILE_PATHS, + DEPS_EXCLUSION_LIST) + def push_data_deps_to_device_dir(device): + device_dir = (constants.TEST_EXECUTABLE_DIR + if test_package.suite_name == 'breakpad_unittests' + else device.GetExternalStoragePath()) + base_setup.PushDataDeps(device, device_dir, test_options) + device_utils.DeviceUtils.parallel(devices).pMap(push_data_deps_to_device_dir) + if i: + i.Clear() + + tests = _GetTests(test_options, test_package, devices) + + # Constructs a new TestRunner with the current options. + def TestRunnerFactory(device, _shard_index): + return test_runner.TestRunner( + test_options, + device, + test_package) + + if test_options.run_disabled: + test_options = test_options._replace( + test_arguments=('%s --gtest_also_run_disabled_tests' % + test_options.test_arguments)) + else: + tests = _FilterDisabledTests(tests, test_options.suite_name, + bool(test_options.gtest_filter)) + if test_options.gtest_filter: + tests = unittest_util.FilterTestNames(tests, test_options.gtest_filter) + + # Coalesce unit tests into a single test per device + if test_options.suite_name not in gtest_test_instance.BROWSER_TEST_SUITES: + num_devices = len(devices) + tests = [':'.join(tests[i::num_devices]) for i in xrange(num_devices)] + tests = [t for t in tests if t] + + return (TestRunnerFactory, tests) diff --git a/build/android/pylib/gtest/test_options.py b/build/android/pylib/gtest/test_options.py new file mode 100644 index 0000000..8bc6996 --- /dev/null +++ b/build/android/pylib/gtest/test_options.py @@ -0,0 +1,19 @@ +# Copyright 2013 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Defines the GTestOptions named tuple.""" + +import collections + +GTestOptions = collections.namedtuple('GTestOptions', [ + 'tool', + 'gtest_filter', + 'run_disabled', + 'test_arguments', + 'timeout', + 'isolate_file_path', + 'suite_name', + 'app_data_files', + 'app_data_file_dir', + 'delete_stale_data']) diff --git a/build/android/pylib/gtest/test_package.py b/build/android/pylib/gtest/test_package.py new file mode 100644 index 0000000..d21360fb9 --- /dev/null +++ b/build/android/pylib/gtest/test_package.py @@ -0,0 +1,80 @@ +# Copyright (c) 2012 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Base class representing GTest test packages.""" +# pylint: disable=R0201 + + +class TestPackage(object): + + """A helper base class for both APK and stand-alone executables. + + Args: + suite_name: Name of the test suite (e.g. base_unittests). + """ + def __init__(self, suite_name): + self.suite_name = suite_name + self.tool = None + + def ClearApplicationState(self, device): + """Clears the application state. + + Args: + device: Instance of DeviceUtils. + """ + raise NotImplementedError('Method must be overridden.') + + def CreateCommandLineFileOnDevice(self, device, test_filter, test_arguments): + """Creates a test runner script and pushes to the device. + + Args: + device: Instance of DeviceUtils. + test_filter: A test_filter flag. + test_arguments: Additional arguments to pass to the test binary. + """ + raise NotImplementedError('Method must be overridden.') + + def GetAllTests(self, device): + """Returns a list of all tests available in the test suite. + + Args: + device: Instance of DeviceUtils. + """ + raise NotImplementedError('Method must be overridden.') + + def GetGTestReturnCode(self, _device): + return None + + def SpawnTestProcess(self, device): + """Spawn the test process. + + Args: + device: Instance of DeviceUtils. + + Returns: + An instance of pexpect spawn class. + """ + raise NotImplementedError('Method must be overridden.') + + def Install(self, device): + """Install the test package to the device. + + Args: + device: Instance of DeviceUtils. + """ + raise NotImplementedError('Method must be overridden.') + + def PullAppFiles(self, device, files, directory): + """Pull application data from the device. + + Args: + device: Instance of DeviceUtils. + files: A list of paths relative to the application data directory to + retrieve from the device. + directory: The host directory to which files should be pulled. + """ + raise NotImplementedError('Method must be overridden.') + + def SetPermissions(self, device): + pass diff --git a/build/android/pylib/gtest/test_package_apk.py b/build/android/pylib/gtest/test_package_apk.py new file mode 100644 index 0000000..c7fa54f --- /dev/null +++ b/build/android/pylib/gtest/test_package_apk.py @@ -0,0 +1,171 @@ +# Copyright (c) 2012 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Defines TestPackageApk to help run APK-based native tests.""" +# pylint: disable=W0212 + +import logging +import os +import sys +import time + +from devil.android import apk_helper +from devil.android import device_errors +from devil.android.sdk import intent +from pylib import constants +from pylib import pexpect +from pylib.gtest import gtest_test_instance +from pylib.gtest.test_package import TestPackage +from pylib.local.device import local_device_gtest_run + + +class TestPackageApk(TestPackage): + """A helper class for running APK-based native tests.""" + + def __init__(self, suite_name): + """ + Args: + suite_name: Name of the test suite (e.g. base_unittests). + """ + TestPackage.__init__(self, suite_name) + self.suite_path = os.path.join( + constants.GetOutDirectory(), '%s_apk' % suite_name, + '%s-debug.apk' % suite_name) + if suite_name == 'content_browsertests': + self._package_info = constants.PACKAGE_INFO['content_browsertests'] + elif suite_name == 'components_browsertests': + self._package_info = constants.PACKAGE_INFO['components_browsertests'] + else: + self._package_info = constants.PACKAGE_INFO['gtest'] + + if suite_name == 'net_unittests': + self._extras = { + 'org.chromium.native_test.NativeTestActivity.RunInSubThread': None + } + else: + self._extras = [] + + def _CreateCommandLineFileOnDevice(self, device, options): + device.WriteFile(self._package_info.cmdline_file, + self.suite_name + ' ' + options) + + def _GetFifo(self): + # The test.fifo path is determined by: + # testing/android/native_test/java/src/org/chromium/native_test/ + # NativeTestActivity.java and + # testing/android/native_test_launcher.cc + return '/data/data/' + self._package_info.package + '/files/test.fifo' + + def _ClearFifo(self, device): + device.RunShellCommand('rm -f ' + self._GetFifo()) + + def _WatchFifo(self, device, timeout, logfile=None): + for i in range(100): + if device.FileExists(self._GetFifo()): + logging.info('Fifo created. Slept for %f secs', i * 0.5) + break + time.sleep(0.5) + else: + raise device_errors.DeviceUnreachableError( + 'Unable to find fifo on device %s ' % self._GetFifo()) + args = ['-s', device.adb.GetDeviceSerial(), 'shell', 'cat', self._GetFifo()] + return pexpect.spawn('adb', args, timeout=timeout, logfile=logfile) + + def _StartActivity(self, device, force_stop=True): + device.StartActivity( + intent.Intent(package=self._package_info.package, + activity=self._package_info.activity, + action='android.intent.action.MAIN', + extras=self._extras), + # No wait since the runner waits for FIFO creation anyway. + blocking=False, + force_stop=force_stop) + + #override + def ClearApplicationState(self, device): + device.ClearApplicationState(self._package_info.package) + # Content shell creates a profile on the sdscard which accumulates cache + # files over time. + if self.suite_name == 'content_browsertests': + try: + device.RunShellCommand( + 'rm -r %s/content_shell' % device.GetExternalStoragePath(), + timeout=60 * 2) + except device_errors.CommandFailedError: + # TODO(jbudorick) Handle this exception appropriately once the + # conversions are done. + pass + elif self.suite_name == 'components_browsertests': + try: + device.RunShellCommand( + 'rm -r %s/components_shell' % device.GetExternalStoragePath(), + timeout=60 * 2) + except device_errors.CommandFailedError: + # TODO(jbudorick) Handle this exception appropriately once the + # conversions are done. + pass + + #override + def CreateCommandLineFileOnDevice(self, device, test_filter, test_arguments): + self._CreateCommandLineFileOnDevice( + device, '--gtest_filter=%s %s' % (test_filter, test_arguments)) + + #override + def GetAllTests(self, device): + self._CreateCommandLineFileOnDevice(device, '--gtest_list_tests') + try: + self.tool.SetupEnvironment() + # Clear and start monitoring logcat. + self._ClearFifo(device) + self._StartActivity(device) + # Wait for native test to complete. + p = self._WatchFifo(device, timeout=30 * self.tool.GetTimeoutScale()) + p.expect('<<ScopedMainEntryLogger') + p.close() + finally: + self.tool.CleanUpEnvironment() + # We need to strip the trailing newline. + content = [line.rstrip() for line in p.before.splitlines()] + return gtest_test_instance.ParseGTestListTests(content) + + #override + def SpawnTestProcess(self, device): + try: + self.tool.SetupEnvironment() + self._ClearFifo(device) + # Doesn't need to stop an Activity because ClearApplicationState() is + # always called before this call and so it is already stopped at this + # point. + self._StartActivity(device, force_stop=False) + finally: + self.tool.CleanUpEnvironment() + logfile = self._NewLineNormalizer(sys.stdout) + return self._WatchFifo(device, timeout=10, logfile=logfile) + + class _NewLineNormalizer(object): + def __init__(self, output): + self._output = output + + def write(self, data): + data = data.replace('\r\r\n', '\n') + self._output.write(data) + + def flush(self): + self._output.flush() + + #override + def Install(self, device): + self.tool.CopyFiles(device) + device.Install(self.suite_path) + + #override + def PullAppFiles(self, device, files, directory): + local_device_gtest_run.PullAppFilesImpl( + device, self._package_info.package, files, directory) + + #override + def SetPermissions(self, device): + permissions = apk_helper.ApkHelper(self.suite_path).GetPermissions() + device.GrantPermissions( + apk_helper.GetPackageName(self.suite_path), permissions) diff --git a/build/android/pylib/gtest/test_package_exe.py b/build/android/pylib/gtest/test_package_exe.py new file mode 100644 index 0000000..d03a596 --- /dev/null +++ b/build/android/pylib/gtest/test_package_exe.py @@ -0,0 +1,163 @@ +# Copyright (c) 2012 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Defines TestPackageExecutable to help run stand-alone executables.""" + +import logging +import os +import posixpath +import sys +import tempfile + +from devil.android import device_errors +from devil.utils import cmd_helper +from pylib import constants +from pylib import pexpect +from pylib.gtest import gtest_test_instance +from pylib.gtest.test_package import TestPackage + + +class TestPackageExecutable(TestPackage): + """A helper class for running stand-alone executables.""" + + _TEST_RUNNER_RET_VAL_FILE = 'gtest_retval' + + def __init__(self, suite_name): + """ + Args: + suite_name: Name of the test suite (e.g. base_unittests). + """ + TestPackage.__init__(self, suite_name) + self.suite_path = os.path.join(constants.GetOutDirectory(), suite_name) + self._symbols_dir = os.path.join(constants.GetOutDirectory(), + 'lib.target') + + #override + def GetGTestReturnCode(self, device): + ret = None + ret_code = 1 # Assume failure if we can't find it + ret_code_file = tempfile.NamedTemporaryFile() + try: + if not device.PullFile( + constants.TEST_EXECUTABLE_DIR + '/' + + TestPackageExecutable._TEST_RUNNER_RET_VAL_FILE, + ret_code_file.name): + logging.critical('Unable to pull gtest ret val file %s', + ret_code_file.name) + raise ValueError + ret_code = file(ret_code_file.name).read() + ret = int(ret_code) + except ValueError: + logging.critical('Error reading gtest ret val file %s [%s]', + ret_code_file.name, ret_code) + ret = 1 + return ret + + @staticmethod + def _AddNativeCoverageExports(device): + # export GCOV_PREFIX set the path for native coverage results + # export GCOV_PREFIX_STRIP indicates how many initial directory + # names to strip off the hardwired absolute paths. + # This value is calculated in buildbot.sh and + # depends on where the tree is built. + # Ex: /usr/local/google/code/chrome will become + # /code/chrome if GCOV_PREFIX_STRIP=3 + try: + depth = os.environ['NATIVE_COVERAGE_DEPTH_STRIP'] + export_string = ('export GCOV_PREFIX="%s/gcov"\n' % + device.GetExternalStoragePath()) + export_string += 'export GCOV_PREFIX_STRIP=%s\n' % depth + return export_string + except KeyError: + logging.info('NATIVE_COVERAGE_DEPTH_STRIP is not defined: ' + 'No native coverage.') + return '' + except device_errors.CommandFailedError: + logging.info('No external storage found: No native coverage.') + return '' + + #override + def ClearApplicationState(self, device): + device.KillAll(self.suite_name, blocking=True, timeout=30, quiet=True) + + #override + def CreateCommandLineFileOnDevice(self, device, test_filter, test_arguments): + tool_wrapper = self.tool.GetTestWrapper() + sh_script_file = tempfile.NamedTemporaryFile() + # We need to capture the exit status from the script since adb shell won't + # propagate to us. + sh_script_file.write( + 'cd %s\n' + '%s' + '%s LD_LIBRARY_PATH=%s/%s_deps %s/%s --gtest_filter=%s %s\n' + 'echo $? > %s' % + (constants.TEST_EXECUTABLE_DIR, + self._AddNativeCoverageExports(device), + tool_wrapper, + constants.TEST_EXECUTABLE_DIR, + self.suite_name, + constants.TEST_EXECUTABLE_DIR, + self.suite_name, + test_filter, test_arguments, + TestPackageExecutable._TEST_RUNNER_RET_VAL_FILE)) + sh_script_file.flush() + cmd_helper.RunCmd(['chmod', '+x', sh_script_file.name]) + device.PushChangedFiles([( + sh_script_file.name, + constants.TEST_EXECUTABLE_DIR + '/chrome_test_runner.sh')]) + logging.info('Conents of the test runner script: ') + for line in open(sh_script_file.name).readlines(): + logging.info(' ' + line.rstrip()) + + #override + def GetAllTests(self, device): + lib_path = posixpath.join( + constants.TEST_EXECUTABLE_DIR, '%s_deps' % self.suite_name) + + cmd = [] + if self.tool.GetTestWrapper(): + cmd.append(self.tool.GetTestWrapper()) + cmd.extend([ + posixpath.join(constants.TEST_EXECUTABLE_DIR, self.suite_name), + '--gtest_list_tests']) + + output = device.RunShellCommand( + cmd, check_return=True, env={'LD_LIBRARY_PATH': lib_path}) + return gtest_test_instance.ParseGTestListTests(output) + + #override + def SpawnTestProcess(self, device): + args = ['adb', '-s', str(device), 'shell', 'sh', + constants.TEST_EXECUTABLE_DIR + '/chrome_test_runner.sh'] + logging.info(args) + return pexpect.spawn(args[0], args[1:], logfile=sys.stdout) + + #override + def Install(self, device): + if self.tool.NeedsDebugInfo(): + target_name = self.suite_path + else: + target_name = self.suite_path + '_stripped' + if not os.path.isfile(target_name): + raise Exception('Did not find %s, build target %s' % + (target_name, self.suite_name + '_stripped')) + + target_mtime = os.stat(target_name).st_mtime + source_mtime = os.stat(self.suite_path).st_mtime + if target_mtime < source_mtime: + raise Exception( + 'stripped binary (%s, timestamp %d) older than ' + 'source binary (%s, timestamp %d), build target %s' % + (target_name, target_mtime, self.suite_path, source_mtime, + self.suite_name + '_stripped')) + + test_binary_path = constants.TEST_EXECUTABLE_DIR + '/' + self.suite_name + device.PushChangedFiles([(target_name, test_binary_path)]) + deps_path = self.suite_path + '_deps' + if os.path.isdir(deps_path): + device.PushChangedFiles([(deps_path, test_binary_path + '_deps')]) + + #override + def PullAppFiles(self, device, files, directory): + pass diff --git a/build/android/pylib/gtest/test_runner.py b/build/android/pylib/gtest/test_runner.py new file mode 100644 index 0000000..28db85b --- /dev/null +++ b/build/android/pylib/gtest/test_runner.py @@ -0,0 +1,218 @@ +# Copyright (c) 2012 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import logging +import os +import re +import tempfile + +from devil.android import device_errors +from devil.android import ports +from devil.android.perf import perf_control +from pylib import pexpect +from pylib.base import base_test_result +from pylib.base import base_test_runner +from pylib.local import local_test_server_spawner + + +# Test case statuses. +RE_RUN = re.compile('\\[ RUN \\] ?(.*)\r\n') +RE_FAIL = re.compile('\\[ FAILED \\] ?(.*?)( \\((\\d+) ms\\))?\r\r\n') +RE_OK = re.compile('\\[ OK \\] ?(.*?)( \\((\\d+) ms\\))?\r\r\n') + +# Test run statuses. +RE_PASSED = re.compile('\\[ PASSED \\] ?(.*)\r\n') +RE_RUNNER_FAIL = re.compile('\\[ RUNNER_FAILED \\] ?(.*)\r\n') +# Signal handlers are installed before starting tests +# to output the CRASHED marker when a crash happens. +RE_CRASH = re.compile('\\[ CRASHED \\](.*)\r\n') + +# Bots that don't output anything for 20 minutes get timed out, so that's our +# hard cap. +_INFRA_STDOUT_TIMEOUT = 20 * 60 + + +def _TestSuiteRequiresMockTestServer(suite_name): + """Returns True if the test suite requires mock test server.""" + tests_require_net_test_server = ['unit_tests', 'net_unittests', + 'components_browsertests', + 'content_unittests', + 'content_browsertests'] + return (suite_name in + tests_require_net_test_server) + +def _TestSuiteRequiresHighPerfMode(suite_name): + """Returns True if the test suite requires high performance mode.""" + return 'perftests' in suite_name + +class TestRunner(base_test_runner.BaseTestRunner): + def __init__(self, test_options, device, test_package): + """Single test suite attached to a single device. + + Args: + test_options: A GTestOptions object. + device: Device to run the tests. + test_package: An instance of TestPackage class. + """ + + super(TestRunner, self).__init__(device, test_options.tool) + + self.test_package = test_package + self.test_package.tool = self.tool + self._test_arguments = test_options.test_arguments + + timeout = test_options.timeout + if timeout == 0: + timeout = 60 + # On a VM (e.g. chromium buildbots), this timeout is way too small. + if os.environ.get('BUILDBOT_SLAVENAME'): + timeout = timeout * 2 + + self._timeout = min(timeout * self.tool.GetTimeoutScale(), + _INFRA_STDOUT_TIMEOUT) + if _TestSuiteRequiresHighPerfMode(self.test_package.suite_name): + self._perf_controller = perf_control.PerfControl(self.device) + + if _TestSuiteRequiresMockTestServer(self.test_package.suite_name): + self._servers = [ + local_test_server_spawner.LocalTestServerSpawner( + ports.AllocateTestServerPort(), self.device, self.tool)] + else: + self._servers = [] + + if test_options.app_data_files: + self._app_data_files = test_options.app_data_files + if test_options.app_data_file_dir: + self._app_data_file_dir = test_options.app_data_file_dir + else: + self._app_data_file_dir = tempfile.mkdtemp() + logging.critical('Saving app files to %s', self._app_data_file_dir) + else: + self._app_data_files = None + self._app_data_file_dir = None + + #override + def InstallTestPackage(self): + self.test_package.Install(self.device) + + def _ParseTestOutput(self, p): + """Process the test output. + + Args: + p: An instance of pexpect spawn class. + + Returns: + A TestRunResults object. + """ + results = base_test_result.TestRunResults() + + log = '' + try: + while True: + full_test_name = None + + found = p.expect([RE_RUN, RE_PASSED, RE_RUNNER_FAIL], + timeout=self._timeout) + if found == 1: # RE_PASSED + break + elif found == 2: # RE_RUNNER_FAIL + break + else: # RE_RUN + full_test_name = p.match.group(1).replace('\r', '') + found = p.expect([RE_OK, RE_FAIL, RE_CRASH], timeout=self._timeout) + log = p.before.replace('\r', '') + if found == 0: # RE_OK + if full_test_name == p.match.group(1).replace('\r', ''): + duration_ms = int(p.match.group(3)) if p.match.group(3) else 0 + results.AddResult(base_test_result.BaseTestResult( + full_test_name, base_test_result.ResultType.PASS, + duration=duration_ms, log=log)) + elif found == 2: # RE_CRASH + results.AddResult(base_test_result.BaseTestResult( + full_test_name, base_test_result.ResultType.CRASH, + log=log)) + break + else: # RE_FAIL + duration_ms = int(p.match.group(3)) if p.match.group(3) else 0 + results.AddResult(base_test_result.BaseTestResult( + full_test_name, base_test_result.ResultType.FAIL, + duration=duration_ms, log=log)) + except pexpect.EOF: + logging.error('Test terminated - EOF') + # We're here because either the device went offline, or the test harness + # crashed without outputting the CRASHED marker (crbug.com/175538). + if not self.device.IsOnline(): + raise device_errors.DeviceUnreachableError( + 'Device %s went offline.' % str(self.device)) + if full_test_name: + results.AddResult(base_test_result.BaseTestResult( + full_test_name, base_test_result.ResultType.CRASH, + log=p.before.replace('\r', ''))) + except pexpect.TIMEOUT: + logging.error('Test terminated after %d second timeout.', + self._timeout) + if full_test_name: + results.AddResult(base_test_result.BaseTestResult( + full_test_name, base_test_result.ResultType.TIMEOUT, + log=p.before.replace('\r', ''))) + finally: + p.close() + + ret_code = self.test_package.GetGTestReturnCode(self.device) + if ret_code: + logging.critical( + 'gtest exit code: %d\npexpect.before: %s\npexpect.after: %s', + ret_code, p.before, p.after) + + return results + + #override + def RunTest(self, test): + test_results = base_test_result.TestRunResults() + if not test: + return test_results, None + + try: + self.test_package.ClearApplicationState(self.device) + self.test_package.CreateCommandLineFileOnDevice( + self.device, test, self._test_arguments) + self.test_package.SetPermissions(self.device) + test_results = self._ParseTestOutput( + self.test_package.SpawnTestProcess(self.device)) + if self._app_data_files: + self.test_package.PullAppFiles(self.device, self._app_data_files, + self._app_data_file_dir) + finally: + for s in self._servers: + s.Reset() + # Calculate unknown test results. + all_tests = set(test.split(':')) + all_tests_ran = set([t.GetName() for t in test_results.GetAll()]) + unknown_tests = all_tests - all_tests_ran + test_results.AddResults( + [base_test_result.BaseTestResult(t, base_test_result.ResultType.UNKNOWN) + for t in unknown_tests]) + retry = ':'.join([t.GetName() for t in test_results.GetNotPass()]) + return test_results, retry + + #override + def SetUp(self): + """Sets up necessary test enviroment for the test suite.""" + super(TestRunner, self).SetUp() + for s in self._servers: + s.SetUp() + if _TestSuiteRequiresHighPerfMode(self.test_package.suite_name): + self._perf_controller.SetHighPerfMode() + self.tool.SetupEnvironment() + + #override + def TearDown(self): + """Cleans up the test enviroment for the test suite.""" + for s in self._servers: + s.TearDown() + if _TestSuiteRequiresHighPerfMode(self.test_package.suite_name): + self._perf_controller.SetDefaultPerfMode() + self.test_package.ClearApplicationState(self.device) + self.tool.CleanUpEnvironment() + super(TestRunner, self).TearDown() diff --git a/build/android/test_runner.py b/build/android/test_runner.py index e324536..8088ad2 100755 --- a/build/android/test_runner.py +++ b/build/android/test_runner.py @@ -31,6 +31,9 @@ from pylib.base import environment_factory from pylib.base import test_dispatcher from pylib.base import test_instance_factory from pylib.base import test_run_factory +from pylib.gtest import gtest_config +from pylib.gtest import setup as gtest_setup +from pylib.gtest import test_options as gtest_test_options from pylib.linker import setup as linker_setup from pylib.host_driven import setup as host_driven_setup from pylib.instrumentation import setup as instrumentation_setup @@ -190,10 +193,15 @@ def AddDeviceOptions(parser): def AddGTestOptions(parser): """Adds gtest options to |parser|.""" + gtest_suites = list(gtest_config.STABLE_TEST_SUITES + + gtest_config.EXPERIMENTAL_TEST_SUITES) + group = parser.add_argument_group('GTest Options') group.add_argument('-s', '--suite', dest='suite_name', nargs='+', metavar='SUITE_NAME', required=True, - help='Executable name of the test suite to run.') + help=('Executable name of the test suite to run. ' + 'Available suites include (but are not limited to): ' + '%s' % ', '.join('"%s"' % s for s in gtest_suites))) group.add_argument('--gtest_also_run_disabled_tests', '--gtest-also-run-disabled-tests', dest='run_disabled', action='store_true', @@ -642,6 +650,44 @@ def AddPythonTestOptions(parser): AddCommonOptions(parser) +def _RunGTests(args, devices): + """Subcommand of RunTestsCommands which runs gtests.""" + exit_code = 0 + for suite_name in args.suite_name: + # TODO(jbudorick): Either deprecate multi-suite or move its handling down + # into the gtest code. + gtest_options = gtest_test_options.GTestOptions( + args.tool, + args.test_filter, + args.run_disabled, + args.test_arguments, + args.timeout, + args.isolate_file_path, + suite_name, + args.app_data_files, + args.app_data_file_dir, + args.delete_stale_data) + runner_factory, tests = gtest_setup.Setup(gtest_options, devices) + + results, test_exit_code = test_dispatcher.RunTests( + tests, runner_factory, devices, shard=True, test_timeout=None, + num_retries=args.num_retries) + + if test_exit_code and exit_code != constants.ERROR_EXIT_CODE: + exit_code = test_exit_code + + report_results.LogFull( + results=results, + test_type='Unit test', + test_package=suite_name, + flakiness_server=args.flakiness_dashboard_server) + + if args.json_results_file: + json_results.GenerateJsonResultsFile(results, args.json_results_file) + + return exit_code + + def _RunLinkerTests(args, devices): """Subcommand of RunTestsCommands which runs linker tests.""" runner_factory, tests = linker_setup.Setup(args, devices) |