summaryrefslogtreecommitdiffstats
path: root/tools/code_coverage/coverage_posix.py
diff options
context:
space:
mode:
authorthakis@chromium.org <thakis@chromium.org@0039d316-1c4b-4281-b951-d872f2087c98>2013-11-14 19:19:50 +0000
committerthakis@chromium.org <thakis@chromium.org@0039d316-1c4b-4281-b951-d872f2087c98>2013-11-14 19:19:50 +0000
commit4c1f5103acbb565ee66a729c56845c1062e4612b (patch)
treed95ef457d4bfcb5b1ba3497170390e3f5aa9566c /tools/code_coverage/coverage_posix.py
parentbad3bef9eb3a3bae971ddee44aa385f786fbbff8 (diff)
downloadchromium_src-4c1f5103acbb565ee66a729c56845c1062e4612b.zip
chromium_src-4c1f5103acbb565ee66a729c56845c1062e4612b.tar.gz
chromium_src-4c1f5103acbb565ee66a729c56845c1062e4612b.tar.bz2
Remove linux coverage scripts.
BUG=318423 TBR=owners Review URL: https://codereview.chromium.org/68193002 git-svn-id: svn://svn.chromium.org/chrome/trunk/src@235208 0039d316-1c4b-4281-b951-d872f2087c98
Diffstat (limited to 'tools/code_coverage/coverage_posix.py')
-rwxr-xr-xtools/code_coverage/coverage_posix.py1266
1 files changed, 0 insertions, 1266 deletions
diff --git a/tools/code_coverage/coverage_posix.py b/tools/code_coverage/coverage_posix.py
deleted file mode 100755
index f4fa56c..0000000
--- a/tools/code_coverage/coverage_posix.py
+++ /dev/null
@@ -1,1266 +0,0 @@
-#!/usr/bin/env python
-# Copyright (c) 2012 The Chromium Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-"""Generate and process code coverage.
-
-TODO(jrg): rename this from coverage_posix.py to coverage_all.py!
-
-Written for and tested on Mac, Linux, and Windows. To use this script
-to generate coverage numbers, please run from within a gyp-generated
-project.
-
-All platforms, to set up coverage:
- cd ...../chromium ; src/tools/gyp/gyp_dogfood -Dcoverage=1 src/build/all.gyp
-
-Run coverage on...
-Mac:
- ( cd src/chrome ; xcodebuild -configuration Debug -target coverage )
-Linux:
- ( cd src/chrome ; hammer coverage )
- # In particular, don't try and run 'coverage' from src/build
-
-
---directory=DIR: specify directory that contains gcda files, and where
- a "coverage" directory will be created containing the output html.
- Example name: ..../chromium/src/xcodebuild/Debug.
- If not specified (e.g. buildbot) we will try and figure it out based on
- other options (e.g. --target and --build-dir; see below).
-
---genhtml: generate html output. If not specified only lcov is generated.
-
---all_unittests: if present, run all files named *_unittests that we
- can find.
-
---fast_test: make the tests run real fast (just for testing)
-
---strict: if a test fails, we continue happily. --strict will cause
- us to die immediately.
-
---trim=False: by default we trim away tests known to be problematic on
- specific platforms. If set to false we do NOT trim out tests.
-
---xvfb=True: By default we use Xvfb to make sure DISPLAY is valid
- (Linux only). if set to False, do not use Xvfb. TODO(jrg): convert
- this script from the compile stage of a builder to a
- RunPythonCommandInBuildDir() command to avoid the need for this
- step.
-
---timeout=SECS: if a subprocess doesn't have output within SECS,
- assume it's a hang. Kill it and give up.
-
---bundles=BUNDLEFILE: a file containing a python list of coverage
- bundles to be eval'd. Example contents of the bundlefile:
- ['../base/base.gyp:base_unittests']
- This is used as part of the coverage bot.
- If no other bundlefile-finding args are used (--target,
- --build-dir), this is assumed to be an absolute path.
- If those args are used, find BUNDLEFILE in a way consistent with
- other scripts launched by buildbot. Example of another script
- launched by buildbot:
- http://src.chromium.org/viewvc/chrome/trunk/tools/buildbot/scripts/slave/runtest.py
-
---target=NAME: specify the build target (e.g. 'Debug' or 'Release').
- This is used by buildbot scripts to help us find the output directory.
- Must be used with --build-dir.
-
---build-dir=DIR: According to buildbot comments, this is the name of
- the directory within the buildbot working directory in which the
- solution, Debug, and Release directories are found.
- It's usually "src/build", but on mac it's $DIR/../xcodebuild and on
- Linux it's $DIR/out.
- This is used by buildbot scripts to help us find the output directory.
- Must be used with --target.
-
---no_exclusions: Do NOT use the exclusion list. This script keeps a
- list of tests known to be problematic under coverage. For example,
- ProcessUtilTest.SpawnChild will crash inside __gcov_fork() when
- using the MacOS 10.6 SDK. Use of --no_exclusions prevents the use
- of this exclusion list.
-
---dont-clear-coverage-data: Normally we clear coverage data from
- previous runs. If this arg is used we do NOT clear the coverage
- data.
-
-Strings after all options are considered tests to run. Test names
-have all text before a ':' stripped to help with gyp compatibility.
-For example, ../base/base.gyp:base_unittests is interpreted as a test
-named "base_unittests".
-"""
-
-import glob
-import logging
-import optparse
-import os
-import Queue
-import re
-import shutil
-import signal
-import subprocess
-import sys
-import tempfile
-import threading
-import time
-import traceback
-
-"""Global list of child PIDs to kill when we die."""
-gChildPIDs = []
-
-"""Exclusion list. Format is
- { platform: { testname: (exclusion1, exclusion2, ... ), ... } }
-
- Platform is a match for sys.platform and can be a list.
- Matching code does an 'if sys.platform in (the key):'.
- Similarly, matching does an 'if testname in thefulltestname:'
-
- The Chromium convention has traditionally been to place the
- exclusion list in a distinct file. Unlike valgrind (which has
- frequent changes when things break and are fixed), the expectation
- here is that exclusions remain relatively constant (e.g. OS bugs).
- If that changes, revisit the decision to place inclusions in this
- script.
-
- Details:
- ProcessUtilTest.SpawnChild: chokes in __gcov_fork on 10.6
- IPCFuzzingTest.MsgBadPayloadArgs: ditto
- PanelBrowserNavigatorTest.NavigateFromCrashedPanel: Fails on coverage bot.
- WebGLConformanceTests.conformance_attribs_gl_enable_vertex_attrib: Fails
- with timeout (45000 ms) exceeded error. crbug.com/143248
- WebGLConformanceTests.conformance_attribs_gl_disabled_vertex_attrib:
- ditto.
- WebGLConformanceTests.conformance_attribs_gl_vertex_attrib_zero_issues:
- ditto.
- WebGLConformanceTests.conformance_attribs_gl_vertex_attrib: ditto.
- WebGLConformanceTests.conformance_attribs_gl_vertexattribpointer_offsets:
- ditto.
- WebGLConformanceTests.conformance_attribs_gl_vertexattribpointer: ditto.
- WebGLConformanceTests.conformance_buffers_buffer_bind_test: After
- disabling WebGLConformanceTests specified above, this test fails when run
- on local machine.
- WebGLConformanceTests.conformance_buffers_buffer_data_array_buffer: ditto.
- WebGLConformanceTests.conformance_buffers_index_validation_copies_indices:
- ditto.
- WebGLConformanceTests.
- conformance_buffers_index_validation_crash_with_buffer_sub_data: ditto.
- WebGLConformanceTests.
- conformance_buffers_index_validation_verifies_too_many_indices: ditto.
- WebGLConformanceTests.
- conformance_buffers_index_validation_with_resized_buffer: ditto.
- WebGLConformanceTests.conformance_canvas_buffer_offscreen_test: ditto.
- WebGLConformanceTests.conformance_canvas_buffer_preserve_test: ditto.
- WebGLConformanceTests.conformance_canvas_canvas_test: ditto.
- WebGLConformanceTests.conformance_canvas_canvas_zero_size: ditto.
- WebGLConformanceTests.
- conformance_canvas_drawingbuffer_static_canvas_test: ditto.
- WebGLConformanceTests.conformance_canvas_drawingbuffer_test: ditto.
- PageCycler*.*: Fails on coverage bot with "Missing test directory
- /....../slave/coverage-dbg-linux/build/src/data/page_cycler/moz" error.
- *FrameRateCompositingTest.*: Fails with
- "FATAL:chrome_content_browser_client.cc(893)] Check failed:
- command_line->HasSwitch(switches::kEnableStatsTable)."
- *FrameRateNoVsyncCanvasInternalTest.*: ditto.
- *FrameRateGpuCanvasInternalTest.*: ditto.
- IndexedDBTest.Perf: Fails with 'Timeout reached in WaitUntilCookieValue'
- error.
- TwoClientPasswordsSyncTest.DeleteAll: Fails on coverage bot.
- MigrationTwoClientTest.MigrationHellWithoutNigori: Fails with timeout
- (45000 ms) exceeded error.
- TwoClientSessionsSyncTest.DeleteActiveSession: ditto.
- MultipleClientSessionsSyncTest.EncryptedAndChanged: ditto.
- MigrationSingleClientTest.AllTypesIndividuallyTriggerNotification: ditto.
- *OldPanelResizeBrowserTest.*: crbug.com/143247
- *OldPanelDragBrowserTest.*: ditto.
- *OldPanelBrowserTest.*: ditto.
- *OldPanelAndDesktopNotificationTest.*: ditto.
- *OldDockedPanelBrowserTest.*: ditto.
- *OldDetachedPanelBrowserTest.*: ditto.
- PanelDragBrowserTest.AttachWithSqueeze: ditto.
- *PanelBrowserTest.*: ditto.
- *DockedPanelBrowserTest.*: ditto.
- *DetachedPanelBrowserTest.*: ditto.
- AutomatedUITest.TheOneAndOnlyTest: crbug.com/143419
- AutomatedUITestBase.DragOut: ditto
-
-"""
-gTestExclusions = {
- 'darwin2': { 'base_unittests': ('ProcessUtilTest.SpawnChild',),
- 'ipc_tests': ('IPCFuzzingTest.MsgBadPayloadArgs',), },
- 'linux2': {
- 'gpu_tests':
- ('WebGLConformanceTests.conformance_attribs_gl_enable_vertex_attrib',
- 'WebGLConformanceTests.'
- 'conformance_attribs_gl_disabled_vertex_attrib',
- 'WebGLConformanceTests.'
- 'conformance_attribs_gl_vertex_attrib_zero_issues',
- 'WebGLConformanceTests.conformance_attribs_gl_vertex_attrib',
- 'WebGLConformanceTests.'
- 'conformance_attribs_gl_vertexattribpointer_offsets',
- 'WebGLConformanceTests.conformance_attribs_gl_vertexattribpointer',
- 'WebGLConformanceTests.conformance_buffers_buffer_bind_test',
- 'WebGLConformanceTests.'
- 'conformance_buffers_buffer_data_array_buffer',
- 'WebGLConformanceTests.'
- 'conformance_buffers_index_validation_copies_indices',
- 'WebGLConformanceTests.'
- 'conformance_buffers_index_validation_crash_with_buffer_sub_data',
- 'WebGLConformanceTests.'
- 'conformance_buffers_index_validation_verifies_too_many_indices',
- 'WebGLConformanceTests.'
- 'conformance_buffers_index_validation_with_resized_buffer',
- 'WebGLConformanceTests.conformance_canvas_buffer_offscreen_test',
- 'WebGLConformanceTests.conformance_canvas_buffer_preserve_test',
- 'WebGLConformanceTests.conformance_canvas_canvas_test',
- 'WebGLConformanceTests.conformance_canvas_canvas_zero_size',
- 'WebGLConformanceTests.'
- 'conformance_canvas_drawingbuffer_static_canvas_test',
- 'WebGLConformanceTests.conformance_canvas_drawingbuffer_test',),
- 'performance_ui_tests':
- ('*PageCycler*.*',
- '*FrameRateCompositingTest.*',
- '*FrameRateNoVsyncCanvasInternalTest.*',
- '*FrameRateGpuCanvasInternalTest.*',
- 'IndexedDBTest.Perf',),
- 'sync_integration_tests':
- ('TwoClientPasswordsSyncTest.DeleteAll',
- 'MigrationTwoClientTest.MigrationHellWithoutNigori',
- 'TwoClientSessionsSyncTest.DeleteActiveSession',
- 'MultipleClientSessionsSyncTest.EncryptedAndChanged',
- 'MigrationSingleClientTest.'
- 'AllTypesIndividuallyTriggerNotification',),
- 'interactive_ui_tests':
- ('*OldPanelResizeBrowserTest.*',
- '*OldPanelDragBrowserTest.*',
- '*OldPanelBrowserTest.*',
- '*OldPanelAndDesktopNotificationTest.*',
- '*OldDockedPanelBrowserTest.*',
- '*OldDetachedPanelBrowserTest.*',
- 'PanelDragBrowserTest.AttachWithSqueeze',
- '*PanelBrowserTest.*',
- '*DockedPanelBrowserTest.*',
- '*DetachedPanelBrowserTest.*',),
- 'automated_ui_tests':
- ('AutomatedUITest.TheOneAndOnlyTest',
- 'AutomatedUITestBase.DragOut',), },
-}
-
-"""Since random tests are failing/hanging on coverage bot, we are enabling
- tests feature by feature. crbug.com/159748
-"""
-gTestInclusions = {
- 'linux2': {
- 'browser_tests':
- (# 'src/chrome/browser/downloads'
- 'SavePageBrowserTest.*',
- 'SavePageAsMHTMLBrowserTest.*',
- 'DownloadQueryTest.*',
- 'DownloadDangerPromptTest.*',
- 'DownloadTest.*',
- # 'src/chrome/browser/net'
- 'CookiePolicyBrowserTest.*',
- 'FtpBrowserTest.*',
- 'LoadTimingObserverTest.*',
- 'PredictorBrowserTest.*',
- 'ProxyBrowserTest.*',
- # 'src/chrome/browser/extensions'
- 'Extension*.*',
- 'WindowOpenPanelDisabledTest.*',
- 'WindowOpenPanelTest.*',
- 'WebstoreStandalone*.*',
- 'CommandLineWebstoreInstall.*',
- 'WebViewTest.*',
- 'RequirementsCheckerBrowserTest.*',
- 'ProcessManagementTest.*',
- 'PlatformAppBrowserTest.*',
- 'PlatformAppDevToolsBrowserTest.*',
- 'LazyBackgroundPageApiTest.*',
- 'IsolatedAppTest.*',
- 'PanelMessagingTest.*',
- 'GeolocationApiTest.*',
- 'ClipboardApiTest.*',
- 'ExecuteScriptApiTest.*',
- 'CalculatorBrowserTest.*',
- 'ChromeAppAPITest.*',
- 'AppApiTest.*',
- 'BlockedAppApiTest.*',
- 'AppBackgroundPageApiTest.*',
- 'WebNavigationApiTest.*',
- 'UsbApiTest.*',
- 'TabCaptureApiTest.*',
- 'SystemInfo*.*',
- 'SyncFileSystemApiTest.*',
- 'SocketApiTest.*',
- 'SerialApiTest.*',
- 'RecordApiTest.*',
- 'PushMessagingApiTest.*',
- 'ProxySettingsApiTest.*',
- 'ExperimentalApiTest.*',
- 'OmniboxApiTest.*',
- 'OffscreenTabsApiTest.*',
- 'NotificationApiTest.*',
- 'MediaGalleriesPrivateApiTest.*',
- 'PlatformAppMediaGalleriesBrowserTest.*',
- 'GetAuthTokenFunctionTest.*',
- 'LaunchWebAuthFlowFunctionTest.*',
- 'FileSystemApiTest.*',
- 'ScriptBadgeApiTest.*',
- 'PageAsBrowserActionApiTest.*',
- 'PageActionApiTest.*',
- 'BrowserActionApiTest.*',
- 'DownloadExtensionTest.*',
- 'DnsApiTest.*',
- 'DeclarativeApiTest.*',
- 'BluetoothApiTest.*',
- 'AllUrlsApiTest.*',
- # 'src/chrome/browser/nacl_host'
- 'nacl_host.*',
- # 'src/chrome/browser/automation'
- 'AutomationMiscBrowserTest.*',
- # 'src/chrome/browser/autofill'
- 'FormStructureBrowserTest.*',
- 'AutofillPopupViewBrowserTest.*',
- 'AutofillTest.*',
- # 'src/chrome/browser/autocomplete'
- 'AutocompleteBrowserTest.*',
- # 'src/chrome/browser/captive_portal'
- 'CaptivePortalBrowserTest.*',
- # 'src/chrome/browser/geolocation'
- 'GeolocationAccessTokenStoreTest.*',
- 'GeolocationBrowserTest.*',
- # 'src/chrome/browser/nacl_host'
- 'NaClGdbTest.*',
- # 'src/chrome/browser/devtools'
- 'DevToolsSanityTest.*',
- 'DevToolsExtensionTest.*',
- 'DevToolsExperimentalExtensionTest.*',
- 'WorkerDevToolsSanityTest.*',
- # 'src/chrome/browser/first_run'
- 'FirstRunBrowserTest.*',
- # 'src/chrome/browser/importer'
- 'ToolbarImporterUtilsTest.*',
- # 'src/chrome/browser/page_cycler'
- 'PageCyclerBrowserTest.*',
- 'PageCyclerCachedBrowserTest.*',
- # 'src/chrome/browser/performance_monitor'
- 'PerformanceMonitorBrowserTest.*',
- 'PerformanceMonitorUncleanExitBrowserTest.*',
- 'PerformanceMonitorSessionRestoreBrowserTest.*',
- # 'src/chrome/browser/prerender'
- 'PrerenderBrowserTest.*',
- 'PrerenderBrowserTestWithNaCl.*',
- 'PrerenderBrowserTestWithExtensions.*',
- 'PrefetchBrowserTest.*',
- 'PrefetchBrowserTestNoPrefetching.*', ),
- },
-}
-
-
-def TerminateSignalHandler(sig, stack):
- """When killed, try and kill our child processes."""
- signal.signal(sig, signal.SIG_DFL)
- for pid in gChildPIDs:
- if 'kill' in os.__all__: # POSIX
- os.kill(pid, sig)
- else:
- subprocess.call(['taskkill.exe', '/PID', str(pid)])
- sys.exit(0)
-
-
-class RunTooLongException(Exception):
- """Thrown when a command runs too long without output."""
- pass
-
-class BadUserInput(Exception):
- """Thrown when arguments from the user are incorrectly formatted."""
- pass
-
-
-class RunProgramThread(threading.Thread):
- """A thread to run a subprocess.
-
- We want to print the output of our subprocess in real time, but also
- want a timeout if there has been no output for a certain amount of
- time. Normal techniques (e.g. loop in select()) aren't cross
- platform enough. the function seems simple: "print output of child, kill it
- if there is no output by timeout. But it was tricky to get this right
- in a x-platform way (see warnings about deadlock on the python
- subprocess doc page).
-
- """
- # Constants in our queue
- PROGRESS = 0
- DONE = 1
-
- def __init__(self, cmd):
- super(RunProgramThread, self).__init__()
- self._cmd = cmd
- self._process = None
- self._queue = Queue.Queue()
- self._retcode = None
-
- def run(self):
- if sys.platform in ('win32', 'cygwin'):
- return self._run_windows()
- else:
- self._run_posix()
-
- def _run_windows(self):
- # We need to save stdout to a temporary file because of a bug on the
- # windows implementation of python which can deadlock while waiting
- # for the IO to complete while writing to the PIPE and the pipe waiting
- # on us and us waiting on the child process.
- stdout_file = tempfile.TemporaryFile()
- try:
- self._process = subprocess.Popen(self._cmd,
- stdin=subprocess.PIPE,
- stdout=stdout_file,
- stderr=subprocess.STDOUT)
- gChildPIDs.append(self._process.pid)
- try:
- # To make sure that the buildbot don't kill us if we run too long
- # without any activity on the console output, we look for progress in
- # the length of the temporary file and we print what was accumulated so
- # far to the output console to make the buildbot know we are making some
- # progress.
- previous_tell = 0
- # We will poll the process until we get a non-None return code.
- self._retcode = None
- while self._retcode is None:
- self._retcode = self._process.poll()
- current_tell = stdout_file.tell()
- if current_tell > previous_tell:
- # Report progress to our main thread so we don't timeout.
- self._queue.put(RunProgramThread.PROGRESS)
- # And print what was accumulated to far.
- stdout_file.seek(previous_tell)
- print stdout_file.read(current_tell - previous_tell),
- previous_tell = current_tell
- # Don't be selfish, let other threads do stuff while we wait for
- # the process to complete.
- time.sleep(0.5)
- # OK, the child process has exited, let's print its output to our
- # console to create debugging logs in case they get to be needed.
- stdout_file.flush()
- stdout_file.seek(previous_tell)
- print stdout_file.read(stdout_file.tell() - previous_tell)
- except IOError, e:
- logging.exception('%s', e)
- pass
- finally:
- stdout_file.close()
-
- # If we get here the process is done.
- gChildPIDs.remove(self._process.pid)
- self._queue.put(RunProgramThread.DONE)
-
- def _run_posix(self):
- """No deadlock problem so use the simple answer. The windows solution
- appears to add extra buffering which we don't want on other platforms."""
- self._process = subprocess.Popen(self._cmd,
- stdout=subprocess.PIPE,
- stderr=subprocess.STDOUT)
- gChildPIDs.append(self._process.pid)
- try:
- while True:
- line = self._process.stdout.readline()
- if not line: # EOF
- break
- print line,
- self._queue.put(RunProgramThread.PROGRESS, True)
- except IOError:
- pass
- # If we get here the process is done.
- gChildPIDs.remove(self._process.pid)
- self._queue.put(RunProgramThread.DONE)
-
- def stop(self):
- self.kill()
-
- def kill(self):
- """Kill our running process if needed. Wait for kill to complete.
-
- Should be called in the PARENT thread; we do not self-kill.
- Returns the return code of the process.
- Safe to call even if the process is dead.
- """
- if not self._process:
- return self.retcode()
- if 'kill' in os.__all__: # POSIX
- os.kill(self._process.pid, signal.SIGKILL)
- else:
- subprocess.call(['taskkill.exe', '/PID', str(self._process.pid)])
- return self.retcode()
-
- def retcode(self):
- """Return the return value of the subprocess.
-
- Waits for process to die but does NOT kill it explicitly.
- """
- if self._retcode == None: # must be none, not 0/False
- self._retcode = self._process.wait()
- return self._retcode
-
- def RunUntilCompletion(self, timeout):
- """Run thread until completion or timeout (in seconds).
-
- Start the thread. Let it run until completion, or until we've
- spent TIMEOUT without seeing output. On timeout throw
- RunTooLongException.
- """
- self.start()
- while True:
- try:
- x = self._queue.get(True, timeout)
- if x == RunProgramThread.DONE:
- return self.retcode()
- except Queue.Empty, e: # timed out
- logging.info('TIMEOUT (%d seconds exceeded with no output): killing' %
- timeout)
- self.kill()
- raise RunTooLongException()
-
-
-class Coverage(object):
- """Doitall class for code coverage."""
-
- def __init__(self, options, args):
- super(Coverage, self).__init__()
- logging.basicConfig(level=logging.DEBUG)
- self.directory = options.directory
- self.options = options
- self.args = args
- self.ConfirmDirectory()
- self.directory_parent = os.path.dirname(self.directory)
- self.output_directory = os.path.join(self.directory, 'coverage')
- if not os.path.exists(self.output_directory):
- os.mkdir(self.output_directory)
- # The "final" lcov-format file
- self.coverage_info_file = os.path.join(self.directory, 'coverage.info')
- # If needed, an intermediate VSTS-format file
- self.vsts_output = os.path.join(self.directory, 'coverage.vsts')
- # Needed for Windows.
- self.src_root = options.src_root
- self.FindPrograms()
- self.ConfirmPlatformAndPaths()
- self.tests = [] # This can be a list of strings, lists or both.
- self.xvfb_pid = 0
- self.test_files = [] # List of files with test specifications.
- self.test_filters = {} # Mapping from testname->--gtest_filter arg.
- logging.info('self.directory: ' + self.directory)
- logging.info('self.directory_parent: ' + self.directory_parent)
-
- def FindInPath(self, program):
- """Find program in our path. Return abs path to it, or None."""
- if not 'PATH' in os.environ:
- logging.fatal('No PATH environment variable?')
- sys.exit(1)
- paths = os.environ['PATH'].split(os.pathsep)
- for path in paths:
- fullpath = os.path.join(path, program)
- if os.path.exists(fullpath):
- return fullpath
- return None
-
- def FindPrograms(self):
- """Find programs we may want to run."""
- if self.IsPosix():
- self.lcov_directory = os.path.join(sys.path[0],
- '../../third_party/lcov/bin')
- self.lcov = os.path.join(self.lcov_directory, 'lcov')
- self.mcov = os.path.join(self.lcov_directory, 'mcov')
- self.genhtml = os.path.join(self.lcov_directory, 'genhtml')
- self.programs = [self.lcov, self.mcov, self.genhtml]
- else:
- # Hack to get the buildbot working.
- os.environ['PATH'] += r';c:\coverage\coverage_analyzer'
- os.environ['PATH'] += r';c:\coverage\performance_tools'
- # (end hack)
- commands = ['vsperfcmd.exe', 'vsinstr.exe', 'coverage_analyzer.exe']
- self.perf = self.FindInPath('vsperfcmd.exe')
- self.instrument = self.FindInPath('vsinstr.exe')
- self.analyzer = self.FindInPath('coverage_analyzer.exe')
- if not self.perf or not self.instrument or not self.analyzer:
- logging.fatal('Could not find Win performance commands.')
- logging.fatal('Commands needed in PATH: ' + str(commands))
- sys.exit(1)
- self.programs = [self.perf, self.instrument, self.analyzer]
-
- def PlatformBuildPrefix(self):
- """Return a platform specific build directory prefix.
-
- This prefix is prepended to the build target (Debug, Release) to
- identify output as relative to the build directory.
- These values are specific to Chromium's use of gyp.
- """
- if self.IsMac():
- return '../xcodebuild'
- if self.IsWindows():
- return ''
- else: # Linux
- return '../out' # assumes make, unlike runtest.py
-
- def ConfirmDirectory(self):
- """Confirm correctness of self.directory.
-
- If it exists, happiness. If not, try and figure it out in a
- manner similar to FindBundlesFile(). The 'figure it out' case
- happens with buildbot where the directory isn't specified
- explicitly.
- """
- if (not self.directory and
- not (self.options.target and self.options.build_dir)):
- logging.fatal('Must use --directory or (--target and --build-dir)')
- sys.exit(1)
-
- if not self.directory:
- self.directory = os.path.join(self.options.build_dir,
- self.PlatformBuildPrefix(),
- self.options.target)
-
- if os.path.exists(self.directory):
- logging.info('Directory: ' + self.directory)
- return
- else:
- logging.fatal('Directory ' +
- self.directory + ' doesn\'t exist')
- sys.exit(1)
-
-
- def FindBundlesFile(self):
- """Find the bundlesfile.
-
- The 'bundles' file can be either absolute path, or (if we are run
- from buildbot) we need to find it based on other hints (--target,
- --build-dir, etc).
- """
- # If no bundle file, no problem!
- if not self.options.bundles:
- return
- # If true, we're buildbot. Form a path.
- # Else assume absolute.
- if self.options.target and self.options.build_dir:
- fullpath = os.path.join(self.options.build_dir,
- self.PlatformBuildPrefix(),
- self.options.target,
- self.options.bundles)
- self.options.bundles = fullpath
-
- if os.path.exists(self.options.bundles):
- logging.info('BundlesFile: ' + self.options.bundles)
- return
- else:
- logging.fatal('bundlefile ' +
- self.options.bundles + ' doesn\'t exist')
- sys.exit(1)
-
-
- def FindTests(self):
- """Find unit tests to run; set self.tests to this list.
-
- Assume all non-option items in the arg list are tests to be run.
- """
- # Before we begin, find the bundles file if not an absolute path.
- self.FindBundlesFile()
-
- # Small tests: can be run in the "chromium" directory.
- # If asked, run all we can find.
- if self.options.all_unittests:
- self.tests += glob.glob(os.path.join(self.directory, '*_unittests'))
- self.tests += glob.glob(os.path.join(self.directory, '*unit_tests'))
- elif self.options.all_browsertests:
- # Run all tests in browser_tests and content_browsertests.
- self.tests += glob.glob(os.path.join(self.directory, 'browser_tests'))
- self.tests += glob.glob(os.path.join(self.directory,
- 'content_browsertests'))
-
- # Tests can come in as args directly, indirectly (through a file
- # of test lists) or as a file of bundles.
- all_testnames = self.args[:] # Copy since we might modify
-
- for test_file in self.options.test_files:
- f = open(test_file)
- for line in f:
- line = re.sub(r"#.*$", "", line)
- line = re.sub(r"\s*", "", line)
- if re.match("\s*$"):
- continue
- all_testnames.append(line)
- f.close()
-
- tests_from_bundles = None
- if self.options.bundles:
- try:
- tests_from_bundles = eval(open(self.options.bundles).read())
- except IOError:
- logging.fatal('IO error in bundle file ' +
- self.options.bundles + ' (doesn\'t exist?)')
- except (NameError, SyntaxError):
- logging.fatal('Parse or syntax error in bundle file ' +
- self.options.bundles)
- if hasattr(tests_from_bundles, '__iter__'):
- all_testnames += tests_from_bundles
- else:
- logging.fatal('Fatal error with bundle file; could not get list from' +
- self.options.bundles)
- sys.exit(1)
-
- # If told explicit tests, run those (after stripping the name as
- # appropriate)
- for testname in all_testnames:
- mo = re.search(r"(.*)\[(.*)\]$", testname)
- gtest_filter = None
- if mo:
- gtest_filter = mo.group(2)
- testname = mo.group(1)
- if ':' in testname:
- testname = testname.split(':')[1]
- # We need 'pyautolib' to run pyauto tests and 'pyautolib' itself is not an
- # executable. So skip this test from adding into coverage_bundles.py.
- if testname == 'pyautolib':
- continue
- self.tests += [os.path.join(self.directory, testname)]
- if gtest_filter:
- self.test_filters[testname] = gtest_filter
-
- # Add 'src/test/functional/pyauto_functional.py' to self.tests.
- # This file with '-v --suite=CODE_COVERAGE' arguments runs all pyauto tests.
- # Pyauto tests are failing randomly on coverage bots. So excluding them.
- # self.tests += [['src/chrome/test/functional/pyauto_functional.py',
- # '-v',
- # '--suite=CODE_COVERAGE']]
-
- # Medium tests?
- # Not sure all of these work yet (e.g. page_cycler_tests)
- # self.tests += glob.glob(os.path.join(self.directory, '*_tests'))
-
- # If needed, append .exe to tests since vsinstr.exe likes it that
- # way.
- if self.IsWindows():
- for ind in range(len(self.tests)):
- test = self.tests[ind]
- test_exe = test + '.exe'
- if not test.endswith('.exe') and os.path.exists(test_exe):
- self.tests[ind] = test_exe
-
- def TrimTests(self):
- """Trim specific tests for each platform."""
- if self.IsWindows():
- return
- # TODO(jrg): remove when not needed
- inclusion = ['unit_tests']
- keep = []
- for test in self.tests:
- for i in inclusion:
- if i in test:
- keep.append(test)
- self.tests = keep
- logging.info('After trimming tests we have ' + ' '.join(self.tests))
- return
- if self.IsLinux():
- # self.tests = filter(lambda t: t.endswith('base_unittests'), self.tests)
- return
- if self.IsMac():
- exclusion = ['automated_ui_tests']
- punted = []
- for test in self.tests:
- for e in exclusion:
- if test.endswith(e):
- punted.append(test)
- self.tests = filter(lambda t: t not in punted, self.tests)
- if punted:
- logging.info('Tests trimmed out: ' + str(punted))
-
- def ConfirmPlatformAndPaths(self):
- """Confirm OS and paths (e.g. lcov)."""
- for program in self.programs:
- if not os.path.exists(program):
- logging.fatal('Program missing: ' + program)
- sys.exit(1)
-
- def Run(self, cmdlist, ignore_error=False, ignore_retcode=None,
- explanation=None):
- """Run the command list; exit fatally on error.
-
- Args:
- cmdlist: a list of commands (e.g. to pass to subprocess.call)
- ignore_error: if True log an error; if False then exit.
- ignore_retcode: if retcode is non-zero, exit unless we ignore.
-
- Returns: process return code.
- Throws: RunTooLongException if the process does not produce output
- within TIMEOUT seconds; timeout is specified as a command line
- option to the Coverage class and is set on init.
- """
- logging.info('Running ' + str(cmdlist))
- t = RunProgramThread(cmdlist)
- retcode = t.RunUntilCompletion(self.options.timeout)
-
- if retcode:
- if ignore_error or retcode == ignore_retcode:
- logging.warning('COVERAGE: %s unhappy but errors ignored %s' %
- (str(cmdlist), explanation or ''))
- else:
- logging.fatal('COVERAGE: %s failed; return code: %d' %
- (str(cmdlist), retcode))
- sys.exit(retcode)
- return retcode
-
- def IsPosix(self):
- """Return True if we are POSIX."""
- return self.IsMac() or self.IsLinux()
-
- def IsMac(self):
- return sys.platform == 'darwin'
-
- def IsLinux(self):
- return sys.platform.startswith('linux')
-
- def IsWindows(self):
- """Return True if we are Windows."""
- return sys.platform in ('win32', 'cygwin')
-
- def ClearData(self):
- """Clear old gcda files and old coverage info files."""
- if self.options.dont_clear_coverage_data:
- print 'Clearing of coverage data NOT performed.'
- return
- print 'Clearing coverage data from previous runs.'
- if os.path.exists(self.coverage_info_file):
- os.remove(self.coverage_info_file)
- if self.IsPosix():
- subprocess.call([self.lcov,
- '--directory', self.directory_parent,
- '--zerocounters'])
- shutil.rmtree(os.path.join(self.directory, 'coverage'))
- if self.options.all_unittests:
- if os.path.exists(os.path.join(self.directory, 'unittests_coverage')):
- shutil.rmtree(os.path.join(self.directory, 'unittests_coverage'))
- elif self.options.all_browsertests:
- if os.path.exists(os.path.join(self.directory,
- 'browsertests_coverage')):
- shutil.rmtree(os.path.join(self.directory, 'browsertests_coverage'))
- else:
- if os.path.exists(os.path.join(self.directory, 'total_coverage')):
- shutil.rmtree(os.path.join(self.directory, 'total_coverage'))
-
- def BeforeRunOneTest(self, testname):
- """Do things before running each test."""
- if not self.IsWindows():
- return
- # Stop old counters if needed
- cmdlist = [self.perf, '-shutdown']
- self.Run(cmdlist, ignore_error=True)
- # Instrument binaries
- for fulltest in self.tests:
- if os.path.exists(fulltest):
- # See http://support.microsoft.com/kb/939818 for details on args
- cmdlist = [self.instrument, '/d:ignorecverr', '/COVERAGE', fulltest]
- self.Run(cmdlist, ignore_retcode=4,
- explanation='OK with a multiple-instrument')
- # Start new counters
- cmdlist = [self.perf, '-start:coverage', '-output:' + self.vsts_output]
- self.Run(cmdlist)
-
- def BeforeRunAllTests(self):
- """Called right before we run all tests."""
- if self.IsLinux() and self.options.xvfb:
- self.StartXvfb()
-
- def GtestFilter(self, fulltest, excl=None):
- """Return a --gtest_filter=BLAH for this test.
-
- Args:
- fulltest: full name of test executable
- exclusions: the exclusions list. Only set in a unit test;
- else uses gTestExclusions.
- Returns:
- String of the form '--gtest_filter=BLAH', or None.
- """
- positive_gfilter_list = []
- negative_gfilter_list = []
-
- # Exclude all flaky, failing, disabled and maybe tests;
- # they don't count for code coverage.
- negative_gfilter_list += ('*.FLAKY_*', '*.FAILS_*',
- '*.DISABLED_*', '*.MAYBE_*')
-
- if not self.options.no_exclusions:
- exclusions = excl or gTestExclusions
- excldict = exclusions.get(sys.platform)
- if excldict:
- for test in excldict.keys():
- # example: if base_unittests in ../blah/blah/base_unittests.exe
- if test in fulltest:
- negative_gfilter_list += excldict[test]
-
- inclusions = gTestInclusions
- include_dict = inclusions.get(sys.platform)
- if include_dict:
- for test in include_dict.keys():
- if test in fulltest:
- positive_gfilter_list += include_dict[test]
-
- fulltest_basename = os.path.basename(fulltest)
- if fulltest_basename in self.test_filters:
- specific_test_filters = self.test_filters[fulltest_basename].split('-')
- if len(specific_test_filters) > 2:
- logging.error('Multiple "-" symbols in filter list: %s' %
- self.test_filters[fulltest_basename])
- raise BadUserInput()
- if len(specific_test_filters) == 2:
- # Remove trailing ':'
- specific_test_filters[0] = specific_test_filters[0][:-1]
-
- if specific_test_filters[0]: # Test for no positive filters.
- positive_gfilter_list += specific_test_filters[0].split(':')
- if len(specific_test_filters) > 1:
- negative_gfilter_list += specific_test_filters[1].split(':')
-
- if not positive_gfilter_list and not negative_gfilter_list:
- return None
-
- result = '--gtest_filter='
- if positive_gfilter_list:
- result += ':'.join(positive_gfilter_list)
- if negative_gfilter_list:
- if positive_gfilter_list: result += ':'
- result += '-' + ':'.join(negative_gfilter_list)
- return result
-
- def RunTests(self):
- """Run all unit tests and generate appropriate lcov files."""
- self.BeforeRunAllTests()
- for fulltest in self.tests:
- if type(fulltest) is str:
- if not os.path.exists(fulltest):
- logging.info(fulltest + ' does not exist')
- if self.options.strict:
- sys.exit(2)
- else:
- logging.info('%s path exists' % fulltest)
- cmdlist = [fulltest, '--gtest_print_time']
-
- # If asked, make this REAL fast for testing.
- if self.options.fast_test:
- logging.info('Running as a FAST test for testing')
- # cmdlist.append('--gtest_filter=RenderWidgetHost*')
- # cmdlist.append('--gtest_filter=CommandLine*')
- cmdlist.append('--gtest_filter=C*')
-
- # Possibly add a test-specific --gtest_filter
- filter = self.GtestFilter(fulltest)
- if filter:
- cmdlist.append(filter)
- elif type(fulltest) is list:
- cmdlist = fulltest
-
- self.BeforeRunOneTest(fulltest)
- logging.info('Running test ' + str(cmdlist))
- try:
- retcode = self.Run(cmdlist, ignore_retcode=True)
- except SystemExit: # e.g. sys.exit() was called somewhere in here
- raise
- except: # can't "except WindowsError" since script runs on non-Windows
- logging.info('EXCEPTION while running a unit test')
- logging.info(traceback.format_exc())
- retcode = 999
- self.AfterRunOneTest(fulltest)
-
- if retcode:
- logging.info('COVERAGE: test %s failed; return code: %d.' %
- (fulltest, retcode))
- if self.options.strict:
- logging.fatal('Test failure is fatal.')
- sys.exit(retcode)
- self.AfterRunAllTests()
-
- def AfterRunOneTest(self, testname):
- """Do things right after running each test."""
- if not self.IsWindows():
- return
- # Stop counters
- cmdlist = [self.perf, '-shutdown']
- self.Run(cmdlist)
- full_output = self.vsts_output + '.coverage'
- shutil.move(full_output, self.vsts_output)
- # generate lcov!
- self.GenerateLcovWindows(testname)
-
- def AfterRunAllTests(self):
- """Do things right after running ALL tests."""
- # On POSIX we can do it all at once without running out of memory.
- # This contrasts with Windows where we must do it after each test.
- if self.IsPosix():
- self.GenerateLcovPosix()
- # Only on Linux do we have the Xvfb step.
- if self.IsLinux() and self.options.xvfb:
- self.StopXvfb()
-
- def StartXvfb(self):
- """Start Xvfb and set an appropriate DISPLAY environment. Linux only.
-
- Copied from http://src.chromium.org/viewvc/chrome/trunk/tools/buildbot/
- scripts/slave/slave_utils.py?view=markup
- with some simplifications (e.g. no need to use xdisplaycheck, save
- pid in var not file, etc)
- """
- logging.info('Xvfb: starting')
- proc = subprocess.Popen(["Xvfb", ":9", "-screen", "0", "1024x768x24",
- "-ac"],
- stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
- self.xvfb_pid = proc.pid
- if not self.xvfb_pid:
- logging.info('Could not start Xvfb')
- return
- os.environ['DISPLAY'] = ":9"
- # Now confirm, giving a chance for it to start if needed.
- logging.info('Xvfb: confirming')
- for test in range(10):
- proc = subprocess.Popen('xdpyinfo >/dev/null', shell=True)
- pid, retcode = os.waitpid(proc.pid, 0)
- if retcode == 0:
- break
- time.sleep(0.5)
- if retcode != 0:
- logging.info('Warning: could not confirm Xvfb happiness')
- else:
- logging.info('Xvfb: OK')
-
- def StopXvfb(self):
- """Stop Xvfb if needed. Linux only."""
- if self.xvfb_pid:
- logging.info('Xvfb: killing')
- try:
- os.kill(self.xvfb_pid, signal.SIGKILL)
- except:
- pass
- del os.environ['DISPLAY']
- self.xvfb_pid = 0
-
- def CopyCoverageFileToDestination(self, coverage_folder):
- coverage_dir = os.path.join(self.directory, coverage_folder)
- if not os.path.exists(coverage_dir):
- os.makedirs(coverage_dir)
- shutil.copyfile(self.coverage_info_file, os.path.join(coverage_dir,
- 'coverage.info'))
-
- def GenerateLcovPosix(self):
- """Convert profile data to lcov on Mac or Linux."""
- start_dir = os.getcwd()
- logging.info('GenerateLcovPosix: start_dir=' + start_dir)
- if self.IsLinux():
- # With Linux/make (e.g. the coverage_run target), the current
- # directory for this command is .../build/src/chrome but we need
- # to be in .../build/src for the relative path of source files
- # to be correct. However, when run from buildbot, the current
- # directory is .../build. Accommodate.
- # On Mac source files are compiled with abs paths so this isn't
- # a problem.
- # This is a bit of a hack. The best answer is to require this
- # script be run in a specific directory for all cases (from
- # Makefile or from buildbot).
- if start_dir.endswith('chrome'):
- logging.info('coverage_posix.py: doing a "cd .." '
- 'to accomodate Linux/make PWD')
- os.chdir('..')
- elif start_dir.endswith('build'):
- logging.info('coverage_posix.py: doing a "cd src" '
- 'to accomodate buildbot PWD')
- os.chdir('src')
- else:
- logging.info('coverage_posix.py: NOT changing directory.')
- elif self.IsMac():
- pass
-
- command = [self.mcov,
- '--directory',
- os.path.join(start_dir, self.directory_parent),
- '--output',
- os.path.join(start_dir, self.coverage_info_file)]
- logging.info('Assembly command: ' + ' '.join(command))
- retcode = subprocess.call(command)
- if retcode:
- logging.fatal('COVERAGE: %s failed; return code: %d' %
- (command[0], retcode))
- if self.options.strict:
- sys.exit(retcode)
- if self.IsLinux():
- os.chdir(start_dir)
-
- # Copy the unittests coverage information to a different folder.
- if self.options.all_unittests:
- self.CopyCoverageFileToDestination('unittests_coverage')
- elif self.options.all_browsertests:
- # Save browsertests only coverage information.
- self.CopyCoverageFileToDestination('browsertests_coverage')
- else:
- # Save the overall coverage information.
- self.CopyCoverageFileToDestination('total_coverage')
-
- if not os.path.exists(self.coverage_info_file):
- logging.fatal('%s was not created. Coverage run failed.' %
- self.coverage_info_file)
- sys.exit(1)
-
- def GenerateLcovWindows(self, testname=None):
- """Convert VSTS format to lcov. Appends coverage data to sum file."""
- lcov_file = self.vsts_output + '.lcov'
- if os.path.exists(lcov_file):
- os.remove(lcov_file)
- # generates the file (self.vsts_output + ".lcov")
-
- cmdlist = [self.analyzer,
- '-sym_path=' + self.directory,
- '-src_root=' + self.src_root,
- '-noxml',
- self.vsts_output]
- self.Run(cmdlist)
- if not os.path.exists(lcov_file):
- logging.fatal('Output file %s not created' % lcov_file)
- sys.exit(1)
- logging.info('Appending lcov for test %s to %s' %
- (testname, self.coverage_info_file))
- size_before = 0
- if os.path.exists(self.coverage_info_file):
- size_before = os.stat(self.coverage_info_file).st_size
- src = open(lcov_file, 'r')
- dst = open(self.coverage_info_file, 'a')
- dst.write(src.read())
- src.close()
- dst.close()
- size_after = os.stat(self.coverage_info_file).st_size
- logging.info('Lcov file growth for %s: %d --> %d' %
- (self.coverage_info_file, size_before, size_after))
-
- def GenerateHtml(self):
- """Convert lcov to html."""
- # TODO(jrg): This isn't happy when run with unit_tests since V8 has a
- # different "base" so V8 includes can't be found in ".". Fix.
- command = [self.genhtml,
- self.coverage_info_file,
- '--output-directory',
- self.output_directory]
- print >>sys.stderr, 'html generation command: ' + ' '.join(command)
- retcode = subprocess.call(command)
- if retcode:
- logging.fatal('COVERAGE: %s failed; return code: %d' %
- (command[0], retcode))
- if self.options.strict:
- sys.exit(retcode)
-
-def CoverageOptionParser():
- """Return an optparse.OptionParser() suitable for Coverage object creation."""
- parser = optparse.OptionParser()
- parser.add_option('-d',
- '--directory',
- dest='directory',
- default=None,
- help='Directory of unit test files')
- parser.add_option('-a',
- '--all_unittests',
- dest='all_unittests',
- default=False,
- help='Run all tests we can find (*_unittests)')
- parser.add_option('-b',
- '--all_browsertests',
- dest='all_browsertests',
- default=False,
- help='Run all tests in browser_tests '
- 'and content_browsertests')
- parser.add_option('-g',
- '--genhtml',
- dest='genhtml',
- default=False,
- help='Generate html from lcov output')
- parser.add_option('-f',
- '--fast_test',
- dest='fast_test',
- default=False,
- help='Make the tests run REAL fast by doing little.')
- parser.add_option('-s',
- '--strict',
- dest='strict',
- default=False,
- help='Be strict and die on test failure.')
- parser.add_option('-S',
- '--src_root',
- dest='src_root',
- default='.',
- help='Source root (only used on Windows)')
- parser.add_option('-t',
- '--trim',
- dest='trim',
- default=True,
- help='Trim out tests? Default True.')
- parser.add_option('-x',
- '--xvfb',
- dest='xvfb',
- default=True,
- help='Use Xvfb for tests? Default True.')
- parser.add_option('-T',
- '--timeout',
- dest='timeout',
- default=5.0 * 60.0,
- type="int",
- help='Timeout before bailing if a subprocess has no output.'
- ' Default is 5min (Buildbot is 10min.)')
- parser.add_option('-B',
- '--bundles',
- dest='bundles',
- default=None,
- help='Filename of bundles for coverage.')
- parser.add_option('--build-dir',
- dest='build_dir',
- default=None,
- help=('Working directory for buildbot build.'
- 'used for finding bundlefile.'))
- parser.add_option('--target',
- dest='target',
- default=None,
- help=('Buildbot build target; '
- 'used for finding bundlefile (e.g. Debug)'))
- parser.add_option('--no_exclusions',
- dest='no_exclusions',
- default=None,
- help=('Disable the exclusion list.'))
- parser.add_option('--dont-clear-coverage-data',
- dest='dont_clear_coverage_data',
- default=False,
- action='store_true',
- help=('Turn off clearing of cov data from a prev run'))
- parser.add_option('-F',
- '--test-file',
- dest="test_files",
- default=[],
- action='append',
- help=('Specify a file from which tests to be run will ' +
- 'be extracted'))
- return parser
-
-
-def main():
- # Print out the args to help someone do it by hand if needed
- print >>sys.stderr, sys.argv
-
- # Try and clean up nice if we're killed by buildbot, Ctrl-C, ...
- signal.signal(signal.SIGINT, TerminateSignalHandler)
- signal.signal(signal.SIGTERM, TerminateSignalHandler)
-
- parser = CoverageOptionParser()
- (options, args) = parser.parse_args()
- if options.all_unittests and options.all_browsertests:
- print 'Error! Can not have all_unittests and all_browsertests together!'
- sys.exit(1)
- coverage = Coverage(options, args)
- coverage.ClearData()
- coverage.FindTests()
- if options.trim:
- coverage.TrimTests()
- coverage.RunTests()
- if options.genhtml:
- coverage.GenerateHtml()
- return 0
-
-
-if __name__ == '__main__':
- sys.exit(main())