path: root/tools/code_coverage/coverage.py

#!/bin/env python
# Copyright (c) 2006-2008 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.


"""Module to setup and generate code coverage data

This module first sets up the environment for code coverage, instruments the 
binaries, runs the tests and collects the code coverage data.


Usage:
  coverage.py --upload=<upload_location> 
              --revision=<revision_number>
              --src_root=<root_of_source_tree>
              [--tools_path=<tools_path>]
"""

import logging
import optparse
import os
import shutil
import subprocess
import sys
import tempfile

import google.logging_utils
import google.process_utils as proc


# The list of binaries that will be instrumented for code coverage
# TODO(niranjan): Re-enable instrumentation of chrome.exe and chrome.dll once we
# resolve the issue where vsinstr.exe is confused while reading symbols.
windows_binaries = [#'chrome.exe',
                    #'chrome.dll',
                    'unit_tests.exe',
                    'automated_ui_tests.exe',
                    'ui_tests.exe',
                    'installer_unittests.exe',
                    'ipc_tests.exe',
                    'memory_test.exe',
                    'page_cycler_tests.exe',
                    'perf_tests.exe',
                    'plugin_tests.exe',
                    'reliability_tests.exe',
                    'security_tests.dll',
                    'startup_tests.exe',
                    'tab_switching_test.exe',
                    'test_shell_tests.exe',
                    'test_shell.exe',
                    'activex_test_control.dll']
                  
# The list of [tests, args] that will be run. 
# Failing tests have been commented out.
# TODO(niranjan): Need to add layout tests that excercise the test shell.
windows_tests = [
                 ['unit_tests.exe', ''],
#                 ['automated_ui_tests.exe', ''], 
                 ['ui_tests.exe', '--no-sandbox'],
                 ['installer_unittests.exe', ''],
                 ['ipc_tests.exe', ''],
                 ['page_cycler_tests.exe', '--gtest_filter=*File --no-sandbox'],
                 ['plugin_tests.exe', '--no-sandbox'],
                 ['reliability_tests.exe', '--no-sandbox'],
                 ['startup_tests.exe', '--no-sandbox'],
                 ['tab_switching_test.exe', '--no-sandbox'],
                 ['test_shell_tests.exe', '']
                ]


def IsWindows():
  """Checks if the current platform is Windows.
  """
  return sys.platform[:3] == 'win'


class Coverage(object):
  """Class to set up and generate code coverage.

  This class contains methods that are useful to set up the environment for 
  code coverage.

  Attributes:
    instrumented: A boolean indicating if all the binaries have been
                  instrumented.
  """

  def __init__(self, 
               revision,
               src_path = None,
               tools_path = None,
               archive=None):
    """Init method for the Coverage class.

    Args:
      revision: Revision number of the Chromium source tree.
      src_path: Location of the Chromium source base.
      tools_path: Location of the Visual Studio Team Tools. (Win32 only)
      archive: Archive location for the intermediate .coverage results.
    """
    google.logging_utils.config_root()
    self.revision = revision
    self.instrumented = False
    self.tools_path = tools_path
    self.src_path = src_path
    self._dir = tempfile.mkdtemp()
    self._archive = archive
  
  
  def SetUp(self, binaries):
    """Set up the platform specific environment and instrument the binaries for
    coverage.

    This method sets up the environment, instruments all the compiled binaries 
    and sets up the code coverage counters.

    Args:
      binaries: List of binaries that need to be instrumented.

    Returns:
      True on success.
      False on error.
    """
    if self.instrumented:
      logging.error('Binaries already instrumented')
      return False
    if IsWindows():
      # Stop all previous instance of VSPerfMon counters
      counters_command = ('%s -shutdown' % 
                          (os.path.join(self.tools_path, 'vsperfcmd.exe')))
      (retcode, output) = proc.RunCommandFull(counters_command,
                                              collect_output=True)
      # TODO(niranjan): Add a check that to verify that the binaries were built
      # using the /PROFILE linker flag.
      if self.tools_path == None:
        logging.error('Could not locate Visual Studio Team Server tools')
        return False
      # Remove trailing slashes
      self.tools_path = self.tools_path.rstrip('\\')
      # Add this to the env PATH.
      os.environ['PATH'] = os.environ['PATH'] + ';' + self.tools_path
      instrument_command = '%s /COVERAGE ' % (os.path.join(self.tools_path,
                                                           'vsinstr.exe'))
      for binary in binaries:
        logging.info('binary = %s' % (binary))
        logging.info('instrument_command = %s' % (instrument_command))
        # Instrument each binary in the list
        binary = os.path.join(self.src_path, 'chrome', 'Release', binary)
        (retcode, output) = proc.RunCommandFull(instrument_command + binary,
                                                collect_output=True)
        # Check if the file has been instrumented correctly.
        if output.pop().rfind('Successfully instrumented') == -1:
          logging.error('Error instrumenting %s' % (binary))
          return False
      # We are now ready to run tests and measure code coverage.
      self.instrumented = True
      return True
      

  def TearDown(self):
    """Tear down method.

    This method shuts down the counters, and cleans up all the intermediate
    artifacts. 
    """
    if self.instrumented == False:
      return
    
    if IsWindows():
      # Stop counters
      counters_command = ('%s -shutdown' % 
                         (os.path.join(self.tools_path, 'vsperfcmd.exe')))
      (retcode, output) = proc.RunCommandFull(counters_command,
                                              collect_output=True)
      logging.info('Counters shut down: %s' % (output))
      # TODO(niranjan): Revert the instrumented binaries to their original
      # versions.
    else:
      return
    if self._archive:
      shutil.copytree(self._dir, os.path.join(self._archive, self.revision))
      logging.info('Archived the .coverage files')
    # Delete all the temp files and folders
    if self._dir != None:
      shutil.rmtree(self._dir, ignore_errors=True)
      logging.info('Cleaned up temporary files and folders')
    # Reset the instrumented flag.
    self.instrumented = False
    

  def RunTest(self, src_root, test):
    """Run tests and collect the .coverage file

    Args: 
      src_root: Path to the root of the source.
      test: Path to the test to be run.

    Returns:
      Path of the intermediate .coverage file on success.
      None on error.
    """
    # Generate the intermediate file name for the coverage results
    test_name = os.path.split(test[0])[1].strip('.exe')
    # test_command = binary + args
    test_command = '%s %s' % (os.path.join(src_root, 
                                           'chrome',
                                           'Release',
                                           test[0]), 
                              test[1])
    
    coverage_file = os.path.join(self._dir, '%s_win32_%s.coverage' % 
                                            (test_name, self.revision))
    logging.info('.coverage file for test %s: %s' % (test_name, coverage_file))

    # After all the binaries have been instrumented, we start the counters.
    counters_command = ('%s -start:coverage -output:%s' %
                        (os.path.join(self.tools_path, 'vsperfcmd.exe'),
                         coverage_file))
    # Here we use subprocess.call() instead of the RunCommandFull because the
    # VSPerfCmd spawns another process before terminating and this confuses
    # the subprocess.Popen() used by RunCommandFull.
    retcode = subprocess.call(counters_command)
    
    # Run the test binary
    logging.info('Executing test %s: ' % test_command)
    (retcode, output) = proc.RunCommandFull(test_command, collect_output=True)
    if retcode != 0: # Return error if the tests fail
      logging.error('One or more tests failed in %s.' % test_command)
      return None
    
    # Stop the counters
    counters_command = ('%s -shutdown' % 
                        (os.path.join(self.tools_path, 'vsperfcmd.exe')))
    (retcode, output) = proc.RunCommandFull(counters_command,
                                            collect_output=True)
    logging.info('Counters shut down: %s' % (output))
    # Return the intermediate .coverage file
    return coverage_file

    
  def Upload(self, list_coverage, upload_path, sym_path=None, src_root=None):
    """Upload the results to the dashboard.

    This method uploads the coverage data to a dashboard where it will be
    processed. On Windows, this method will first convert the .coverage file to
    the lcov format. This method needs to be called before the TearDown method.

    Args:
      list_coverage: The list of coverage data files to consoliate and upload.
      upload_path: Destination where the coverage data will be processed.
      sym_path: Symbol path for the build (Win32 only)
      src_root: Root folder of the source tree (Win32 only)
    
    Returns:
      True on success.
      False on failure.
    """
    if upload_path == None:
      logging.info('Upload path not specified. Will not convert to LCOV')
      return True
    
    if IsWindows():
      # Stop counters
      counters_command = ('%s -shutdown' % 
                          (os.path.join(self.tools_path, 'vsperfcmd.exe')))
      (retcode, output) = proc.RunCommandFull(counters_command,
                                              collect_output=True)
      logging.info('Counters shut down: %s' % (output))
      lcov_file = os.path.join(upload_path, 'chrome_win32_%s.lcov' % 
                                            (self.revision))
      lcov = open(lcov_file, 'w')
      for coverage_file in list_coverage:
        # Convert the intermediate .coverage file to lcov format
        if self.tools_path == None:
          logging.error('Lcov converter tool not found')
          return False
        self.tools_path = self.tools_path.rstrip('\\')
        convert_command = ('%s -sym_path=%s -src_root=%s %s' % 
                           (os.path.join(self.tools_path, 
                                         'coverage_analyzer.exe'),
                           sym_path,
                           src_root,
                           coverage_file))
        (retcode, output) = proc.RunCommandFull(convert_command,
                                                collect_output=True)
        # TODO(niranjan): Fix this to check for the correct return code.
#        if output != 0:
#          logging.error('Conversion to LCOV failed. Exiting.')
        tmp_lcov_file = coverage_file + '.lcov'
        logging.info('Conversion to lcov complete for %s' % (coverage_file))
        # Now append this .lcov file to the cumulative lcov file
        logging.info('Consolidating LCOV file: %s' % (tmp_lcov_file))
        tmp_lcov = open(tmp_lcov_file, 'r')
        lcov.write(tmp_lcov.read())
        tmp_lcov.close()
      lcov.close()
      logging.info('LCOV file uploaded to %s' % (upload_path))


def main():
  # Command line parsing
  parser = optparse.OptionParser()
  # Path where the .coverage to .lcov converter tools are stored.
  parser.add_option('-t',
                    '--tools_path',
                    dest='tools_path',
                    default=None,
                    help='Location of the coverage tools (windows only)')
  parser.add_option('-u',
                    '--upload',
                    dest='upload_path',
                    default=None,
                    help='Location where the results should be uploaded')
  # We need the revision number so that we can generate the output file of the
  # format chrome_<platform>_<revision>.lcov
  parser.add_option('-r',
                    '--revision',
                    dest='revision',
                    default=None,
                    help='Revision number of the Chromium source repo')
  # Root of the source tree. Needed for converting the generated .coverage file
  # on Windows to the open source lcov format.
  parser.add_option('-s',
                    '--src_root',
                    dest='src_root',
                    default=None,
                    help='Root of the source repository')
  parser.add_option('-a',
                    '--archive',
                    dest='archive',
                    default=None,
                    help='Archive location of the intermediate .coverage data')

  (options, args) = parser.parse_args()
  
  if options.revision == None:
    parser.error('Revision number not specified')
  if options.src_root == None:
    parser.error('Source root not specified')
   
  if IsWindows():
    # Initialize coverage
    cov = Coverage(options.revision,
                   options.src_root,
                   options.tools_path,
                   options.archive)
    list_coverage = []
    # Instrument the binaries
    if cov.SetUp(windows_binaries):
      # Run all the tests
      for test in windows_tests:
        coverage = cov.RunTest(options.src_root, test)
        if coverage == None: # Indicate failure to the buildbots.
          return 1
        # Collect the intermediate file
        list_coverage.append(coverage)
    else: 
      logging.error('Error during instrumentation.')
      sys.exit(1)

    cov.Upload(list_coverage,
               options.upload_path,
               os.path.join(options.src_root, 'chrome', 'Release'),
               options.src_root)
    cov.TearDown()


if __name__ == '__main__':
  sys.exit(main())