#!/bin/env python
# Copyright (c) 2006-2008 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Module to setup and generate code coverage data
This module first sets up the environment for code coverage, instruments the
binaries, runs the tests and collects the code coverage data.
Usage:
coverage.py --upload=<upload_location>
--revision=<revision_number>
--src_root=<root_of_source_tree>
[--tools_path=<tools_path>]
"""
import logging
import optparse
import os
import shutil
import subprocess
import sys
import tempfile
import google.logging_utils
import google.process_utils as proc
# The list of binaries that will be instrumented for code coverage
# TODO(niranjan): Re-enable instrumentation of chrome.exe and chrome.dll once we
# resolve the issue where vsinstr.exe is confused while reading symbols.
windows_binaries = [#'chrome.exe',
#'chrome.dll',
'unit_tests.exe',
'automated_ui_tests.exe',
'ui_tests.exe',
'installer_unittests.exe',
'ipc_tests.exe',
'memory_test.exe',
'page_cycler_tests.exe',
'perf_tests.exe',
'plugin_tests.exe',
'reliability_tests.exe',
'security_tests.dll',
'startup_tests.exe',
'tab_switching_test.exe',
'test_shell_tests.exe',
'test_shell.exe',
'activex_test_control.dll']
# The list of [tests, args] that will be run.
# Failing tests have been commented out.
# TODO(niranjan): Need to add layout tests that excercise the test shell.
windows_tests = [
['unit_tests.exe', ''],
# ['automated_ui_tests.exe', ''],
['ui_tests.exe', '--no-sandbox'],
['installer_unittests.exe', ''],
['ipc_tests.exe', ''],
['page_cycler_tests.exe', '--gtest_filter=*File --no-sandbox'],
['plugin_tests.exe', '--no-sandbox'],
['reliability_tests.exe', '--no-sandbox'],
['startup_tests.exe', '--no-sandbox'],
['tab_switching_test.exe', '--no-sandbox'],
['test_shell_tests.exe', '']
]
def IsWindows():
"""Checks if the current platform is Windows.
"""
return sys.platform[:3] == 'win'
class Coverage(object):
"""Class to set up and generate code coverage.
This class contains methods that are useful to set up the environment for
code coverage.
Attributes:
instrumented: A boolean indicating if all the binaries have been
instrumented.
"""
def __init__(self,
revision,
src_path = None,
tools_path = None,
archive=None):
"""Init method for the Coverage class.
Args:
revision: Revision number of the Chromium source tree.
src_path: Location of the Chromium source base.
tools_path: Location of the Visual Studio Team Tools. (Win32 only)
archive: Archive location for the intermediate .coverage results.
"""
google.logging_utils.config_root()
self.revision = revision
self.instrumented = False
self.tools_path = tools_path
self.src_path = src_path
self._dir = tempfile.mkdtemp()
self._archive = archive
def SetUp(self, binaries):
"""Set up the platform specific environment and instrument the binaries for
coverage.
This method sets up the environment, instruments all the compiled binaries
and sets up the code coverage counters.
Args:
binaries: List of binaries that need to be instrumented.
Returns:
True on success.
False on error.
"""
if self.instrumented:
logging.error('Binaries already instrumented')
return False
if IsWindows():
# Stop all previous instance of VSPerfMon counters
counters_command = ('%s -shutdown' %
(os.path.join(self.tools_path, 'vsperfcmd.exe')))
(retcode, output) = proc.RunCommandFull(counters_command,
collect_output=True)
# TODO(niranjan): Add a check that to verify that the binaries were built
# using the /PROFILE linker flag.
if self.tools_path == None:
logging.error('Could not locate Visual Studio Team Server tools')
return False
# Remove trailing slashes
self.tools_path = self.tools_path.rstrip('\\')
# Add this to the env PATH.
os.environ['PATH'] = os.environ['PATH'] + ';' + self.tools_path
instrument_command = '%s /COVERAGE ' % (os.path.join(self.tools_path,
'vsinstr.exe'))
for binary in binaries:
logging.info('binary = %s' % (binary))
logging.info('instrument_command = %s' % (instrument_command))
# Instrument each binary in the list
binary = os.path.join(self.src_path, 'chrome', 'Release', binary)
(retcode, output) = proc.RunCommandFull(instrument_command + binary,
collect_output=True)
# Check if the file has been instrumented correctly.
if output.pop().rfind('Successfully instrumented') == -1:
logging.error('Error instrumenting %s' % (binary))
return False
# We are now ready to run tests and measure code coverage.
self.instrumented = True
return True
def TearDown(self):
"""Tear down method.
This method shuts down the counters, and cleans up all the intermediate
artifacts.
"""
if self.instrumented == False:
return
if IsWindows():
# Stop counters
counters_command = ('%s -shutdown' %
(os.path.join(self.tools_path, 'vsperfcmd.exe')))
(retcode, output) = proc.RunCommandFull(counters_command,
collect_output=True)
logging.info('Counters shut down: %s' % (output))
# TODO(niranjan): Revert the instrumented binaries to their original
# versions.
else:
return
if self._archive:
shutil.copytree(self._dir, os.path.join(self._archive, self.revision))
logging.info('Archived the .coverage files')
# Delete all the temp files and folders
if self._dir != None:
shutil.rmtree(self._dir, ignore_errors=True)
logging.info('Cleaned up temporary files and folders')
# Reset the instrumented flag.
self.instrumented = False
def RunTest(self, src_root, test):
"""Run tests and collect the .coverage file
Args:
src_root: Path to the root of the source.
test: Path to the test to be run.
Returns:
Path of the intermediate .coverage file on success.
None on error.
"""
# Generate the intermediate file name for the coverage results
test_name = os.path.split(test[0])[1].strip('.exe')
# test_command = binary + args
test_command = '%s %s' % (os.path.join(src_root,
'chrome',
'Release',
test[0]),
test[1])
coverage_file = os.path.join(self._dir, '%s_win32_%s.coverage' %
(test_name, self.revision))
logging.info('.coverage file for test %s: %s' % (test_name, coverage_file))
# After all the binaries have been instrumented, we start the counters.
counters_command = ('%s -start:coverage -output:%s' %
(os.path.join(self.tools_path, 'vsperfcmd.exe'),
coverage_file))
# Here we use subprocess.call() instead of the RunCommandFull because the
# VSPerfCmd spawns another process before terminating and this confuses
# the subprocess.Popen() used by RunCommandFull.
retcode = subprocess.call(counters_command)
# Run the test binary
logging.info('Executing test %s: ' % test_command)
(retcode, output) = proc.RunCommandFull(test_command, collect_output=True)
if retcode != 0: # Return error if the tests fail
logging.error('One or more tests failed in %s.' % test_command)
return None
# Stop the counters
counters_command = ('%s -shutdown' %
(os.path.join(self.tools_path, 'vsperfcmd.exe')))
(retcode, output) = proc.RunCommandFull(counters_command,
collect_output=True)
logging.info('Counters shut down: %s' % (output))
# Return the intermediate .coverage file
return coverage_file
def Upload(self, list_coverage, upload_path, sym_path=None, src_root=None):
"""Upload the results to the dashboard.
This method uploads the coverage data to a dashboard where it will be
processed. On Windows, this method will first convert the .coverage file to
the lcov format. This method needs to be called before the TearDown method.
Args:
list_coverage: The list of coverage data files to consoliate and upload.
upload_path: Destination where the coverage data will be processed.
sym_path: Symbol path for the build (Win32 only)
src_root: Root folder of the source tree (Win32 only)
Returns:
True on success.
False on failure.
"""
if upload_path == None:
logging.info('Upload path not specified. Will not convert to LCOV')
return True
if IsWindows():
# Stop counters
counters_command = ('%s -shutdown' %
(os.path.join(self.tools_path, 'vsperfcmd.exe')))
(retcode, output) = proc.RunCommandFull(counters_command,
collect_output=True)
logging.info('Counters shut down: %s' % (output))
lcov_file = os.path.join(upload_path, 'chrome_win32_%s.lcov' %
(self.revision))
lcov = open(lcov_file, 'w')
for coverage_file in list_coverage:
# Convert the intermediate .coverage file to lcov format
if self.tools_path == None:
logging.error('Lcov converter tool not found')
return False
self.tools_path = self.tools_path.rstrip('\\')
convert_command = ('%s -sym_path=%s -src_root=%s %s' %
(os.path.join(self.tools_path,
'coverage_analyzer.exe'),
sym_path,
src_root,
coverage_file))
(retcode, output) = proc.RunCommandFull(convert_command,
collect_output=True)
# TODO(niranjan): Fix this to check for the correct return code.
# if output != 0:
# logging.error('Conversion to LCOV failed. Exiting.')
tmp_lcov_file = coverage_file + '.lcov'
logging.info('Conversion to lcov complete for %s' % (coverage_file))
# Now append this .lcov file to the cumulative lcov file
logging.info('Consolidating LCOV file: %s' % (tmp_lcov_file))
tmp_lcov = open(tmp_lcov_file, 'r')
lcov.write(tmp_lcov.read())
tmp_lcov.close()
lcov.close()
logging.info('LCOV file uploaded to %s' % (upload_path))
def main():
# Command line parsing
parser = optparse.OptionParser()
# Path where the .coverage to .lcov converter tools are stored.
parser.add_option('-t',
'--tools_path',
dest='tools_path',
default=None,
help='Location of the coverage tools (windows only)')
parser.add_option('-u',
'--upload',
dest='upload_path',
default=None,
help='Location where the results should be uploaded')
# We need the revision number so that we can generate the output file of the
# format chrome_<platform>_<revision>.lcov
parser.add_option('-r',
'--revision',
dest='revision',
default=None,
help='Revision number of the Chromium source repo')
# Root of the source tree. Needed for converting the generated .coverage file
# on Windows to the open source lcov format.
parser.add_option('-s',
'--src_root',
dest='src_root',
default=None,
help='Root of the source repository')
parser.add_option('-a',
'--archive',
dest='archive',
default=None,
help='Archive location of the intermediate .coverage data')
(options, args) = parser.parse_args()
if options.revision == None:
parser.error('Revision number not specified')
if options.src_root == None:
parser.error('Source root not specified')
if IsWindows():
# Initialize coverage
cov = Coverage(options.revision,
options.src_root,
options.tools_path,
options.archive)
list_coverage = []
# Instrument the binaries
if cov.SetUp(windows_binaries):
# Run all the tests
for test in windows_tests:
coverage = cov.RunTest(options.src_root, test)
if coverage == None: # Indicate failure to the buildbots.
return 1
# Collect the intermediate file
list_coverage.append(coverage)
else:
logging.error('Error during instrumentation.')
sys.exit(1)
cov.Upload(list_coverage,
options.upload_path,
os.path.join(options.src_root, 'chrome', 'Release'),
options.src_root)
cov.TearDown()
if __name__ == '__main__':
sys.exit(main())