diff options
author | phajdan.jr <phajdan.jr@chromium.org> | 2015-08-18 04:46:06 -0700 |
---|---|---|
committer | Commit bot <commit-bot@chromium.org> | 2015-08-18 11:46:52 +0000 |
commit | 39a1c284b008e3db5cb916ba489a2b65e04c21cc (patch) | |
tree | 370ffd1f70b2d22841b98e0f260cac6f61ddca32 | |
parent | 28fccd5cbdec7e4c1a38d07926591da41331d7d8 (diff) | |
download | chromium_src-39a1c284b008e3db5cb916ba489a2b65e04c21cc.zip chromium_src-39a1c284b008e3db5cb916ba489a2b65e04c21cc.tar.gz chromium_src-39a1c284b008e3db5cb916ba489a2b65e04c21cc.tar.bz2 |
runtest.py: remove legacy test-results related code
See https://goto.google.com/vdhuz for context (Google-internal).
BUG=506498
Review URL: https://codereview.chromium.org/1302443003
Cr-Commit-Position: refs/heads/master@{#343887}
-rw-r--r-- | infra/scripts/legacy/scripts/common/chromium_utils.py | 280 | ||||
-rwxr-xr-x | infra/scripts/legacy/scripts/common/env.py | 439 | ||||
-rw-r--r-- | infra/scripts/legacy/scripts/common/url_helper.py | 60 | ||||
-rw-r--r-- | infra/scripts/legacy/scripts/slave/bootstrap.py | 47 | ||||
-rw-r--r-- | infra/scripts/legacy/scripts/slave/gtest/__init__.py | 0 | ||||
-rw-r--r-- | infra/scripts/legacy/scripts/slave/gtest/json_results_generator.py | 255 | ||||
-rw-r--r-- | infra/scripts/legacy/scripts/slave/gtest/networktransaction.py | 46 | ||||
-rw-r--r-- | infra/scripts/legacy/scripts/slave/gtest/test_result.py | 42 | ||||
-rw-r--r-- | infra/scripts/legacy/scripts/slave/gtest/test_results_uploader.py | 32 | ||||
-rwxr-xr-x | infra/scripts/legacy/scripts/slave/gtest_slave_utils.py | 103 | ||||
-rwxr-xr-x | infra/scripts/legacy/scripts/slave/runtest.py | 114 | ||||
-rw-r--r-- | infra/scripts/legacy/site_config/config.py | 18 | ||||
-rw-r--r-- | infra/scripts/legacy/site_config/config_bootstrap.py | 128 | ||||
-rw-r--r-- | infra/scripts/legacy/site_config/config_default.py | 230 |
14 files changed, 0 insertions, 1794 deletions
diff --git a/infra/scripts/legacy/scripts/common/chromium_utils.py b/infra/scripts/legacy/scripts/common/chromium_utils.py index 89ebf8b..9371cb7 100644 --- a/infra/scripts/legacy/scripts/common/chromium_utils.py +++ b/infra/scripts/legacy/scripts/common/chromium_utils.py @@ -23,8 +23,6 @@ import threading import time import traceback -from common import env - BUILD_DIR = os.path.realpath(os.path.join( os.path.dirname(__file__), os.pardir, os.pardir)) @@ -472,149 +470,6 @@ def GetCommandOutput(command): return output -def ListMasters(cue='master.cfg', include_public=True, include_internal=True): - """Returns all the masters found.""" - # Look for "internal" masters first. - path_internal = os.path.join( - BUILD_DIR, os.pardir, 'build_internal', 'masters/*/' + cue) - path = os.path.join(BUILD_DIR, 'masters/*/' + cue) - filenames = [] - if include_public: - filenames += glob.glob(path) - if include_internal: - filenames += glob.glob(path_internal) - return [os.path.abspath(os.path.dirname(f)) for f in filenames] - - -def MasterPath(mastername, include_public=True, include_internal=True): - path = os.path.join(BUILD_DIR, 'masters', 'master.%s' % mastername) - path_internal = os.path.join( - BUILD_DIR, os.pardir, 'build_internal', 'masters', - 'master.%s' % mastername) - if include_public and os.path.isdir(path): - return path - if include_internal and os.path.isdir(path_internal): - return path_internal - raise LookupError('Path for master %s not found' % mastername) - - -def ListMastersWithSlaves(include_public=True, include_internal=True): - masters_path = ListMasters('builders.pyl', include_public, include_internal) - masters_path.extend(ListMasters('slaves.cfg', include_public, - include_internal)) - return masters_path - - -def GetSlavesFromMasterPath(path, fail_hard=False): - builders_path = os.path.join(path, 'builders.pyl') - if os.path.exists(builders_path): - return GetSlavesFromBuildersFile(builders_path) - return RunSlavesCfg(os.path.join(path, 'slaves.cfg'), fail_hard=fail_hard) - - -def GetAllSlaves(fail_hard=False, include_public=True, include_internal=True): - """Return all slave objects from masters.""" - slaves = [] - for master in ListMastersWithSlaves(include_public, include_internal): - cur_slaves = GetSlavesFromMasterPath(master, fail_hard) - for slave in cur_slaves: - slave['mastername'] = os.path.basename(master) - slaves.extend(cur_slaves) - return slaves - - -def GetActiveSubdir(): - """Get current checkout's subdir, if checkout uses subdir layout.""" - rootdir, subdir = os.path.split(os.path.dirname(BUILD_DIR)) - if subdir != 'b' and os.path.basename(rootdir) == 'c': - return subdir - - -def GetActiveSlavename(): - slavename = os.getenv('TESTING_SLAVENAME') - if not slavename: - slavename = socket.getfqdn().split('.', 1)[0].lower() - subdir = GetActiveSubdir() - if subdir: - return '%s#%s' % (slavename, subdir) - return slavename - - -def EntryToSlaveName(entry): - """Produces slave name from the slaves config dict.""" - name = entry.get('slavename') or entry.get('hostname') - if 'subdir' in entry: - return '%s#%s' % (name, entry['subdir']) - return name - - -def GetActiveMaster(slavename=None, default=None): - """Returns the name of the Active master serving the current host. - - Parse all of the active masters with slaves matching the current hostname - and optional slavename. Returns |default| if no match found. - """ - slavename = slavename or GetActiveSlavename() - for slave in GetAllSlaves(): - if slavename == EntryToSlaveName(slave): - return slave['master'] - return default - - -@contextmanager -def MasterEnvironment(master_dir): - """Context manager that enters an enviornment similar to a master's. - - This involves: - - Modifying 'sys.path' to include paths available to the master. - - Changing directory (via os.chdir()) to the master's base directory. - - These changes will be reverted after the context manager completes. - - Args: - master_dir: (str) The master's base directory. - """ - master_dir = os.path.abspath(master_dir) - - # Setup a 'sys.path' that is adequate for loading 'slaves.cfg'. - old_cwd = os.getcwd() - - with env.GetInfraPythonPath(master_dir=master_dir).Enter(): - try: - os.chdir(master_dir) - yield - finally: - os.chdir(old_cwd) - - -def ParsePythonCfg(cfg_filepath, fail_hard=False): - """Retrieves data from a python config file.""" - if not os.path.exists(cfg_filepath): - return None - - # Execute 'slaves.sfg' in the master path environment. - with MasterEnvironment(os.path.dirname(os.path.abspath(cfg_filepath))): - try: - local_vars = {} - execfile(os.path.join(cfg_filepath), local_vars) - del local_vars['__builtins__'] - return local_vars - except Exception as e: - # pylint: disable=C0323 - print >>sys.stderr, 'An error occurred while parsing %s: %s' % ( - cfg_filepath, e) - print >>sys.stderr, traceback.format_exc() # pylint: disable=C0323 - if fail_hard: - raise - return {} - - -def RunSlavesCfg(slaves_cfg, fail_hard=False): - """Runs slaves.cfg in a consistent way.""" - slave_config = ParsePythonCfg(slaves_cfg, fail_hard=fail_hard) or {} - return slave_config.get('slaves', []) - - def convert_json(option, _, value, parser): """Provide an OptionParser callback to unmarshal a JSON string.""" setattr(parser.values, option.dest, json.loads(value)) @@ -640,138 +495,3 @@ def AddPropertiesOptions(option_parser): callback=convert_json, type='string', nargs=1, default={}, help='factory properties in JSON format') - - -def ReadJsonAsUtf8(filename=None, text=None): - """Read a json file or string and output a dict. - - This function is different from json.load and json.loads in that it - returns utf8-encoded string for keys and values instead of unicode. - - Args: - filename: path of a file to parse - text: json string to parse - - If both 'filename' and 'text' are provided, 'filename' is used. - """ - def _decode_list(data): - rv = [] - for item in data: - if isinstance(item, unicode): - item = item.encode('utf-8') - elif isinstance(item, list): - item = _decode_list(item) - elif isinstance(item, dict): - item = _decode_dict(item) - rv.append(item) - return rv - - def _decode_dict(data): - rv = {} - for key, value in data.iteritems(): - if isinstance(key, unicode): - key = key.encode('utf-8') - if isinstance(value, unicode): - value = value.encode('utf-8') - elif isinstance(value, list): - value = _decode_list(value) - elif isinstance(value, dict): - value = _decode_dict(value) - rv[key] = value - return rv - - if filename: - with open(filename, 'rb') as f: - return json.load(f, object_hook=_decode_dict) - if text: - return json.loads(text, object_hook=_decode_dict) - - -def DatabaseSetup(buildmaster_config, require_dbconfig=False): - """Read database credentials in the master directory.""" - if os.path.isfile('.dbconfig'): - values = {} - execfile('.dbconfig', values) - if 'password' not in values: - raise Exception('could not get db password') - - buildmaster_config['db_url'] = 'postgresql://%s:%s@%s/%s' % ( - values['username'], values['password'], - values.get('hostname', 'localhost'), values['dbname']) - else: - assert not require_dbconfig - - -def ReadBuildersFile(builders_path): - with open(builders_path) as fp: - contents = fp.read() - return ParseBuildersFileContents(builders_path, contents) - - -def ParseBuildersFileContents(path, contents): - builders = ast.literal_eval(contents) - - # Set some additional derived fields that are derived from the - # file's location in the filesystem. - basedir = os.path.dirname(os.path.abspath(path)) - master_dirname = os.path.basename(basedir) - master_name_comps = master_dirname.split('.')[1:] - buildbot_path = '.'.join(master_name_comps) - master_classname = ''.join(c[0].upper() + c[1:] for c in master_name_comps) - builders['master_dirname'] = master_dirname - builders.setdefault('master_classname', master_classname) - builders.setdefault('buildbot_url', - 'https://build.chromium.org/p/%s/' % buildbot_path) - - builders.setdefault('buildbucket_bucket', None) - builders.setdefault('service_account_file', None) - - # The _str fields are printable representations of Python values: - # if builders['foo'] == "hello", then builders['foo_str'] == "'hello'". - # This allows them to be read back in by Python scripts properly. - builders['buildbucket_bucket_str'] = repr(builders['buildbucket_bucket']) - builders['service_account_file_str'] = repr(builders['service_account_file']) - - return builders - - -def GetSlavesFromBuildersFile(builders_path): - """Read builders_path and return a list of slave dicts.""" - builders = ReadBuildersFile(builders_path) - return GetSlavesFromBuilders(builders) - - -def GetSlavesFromBuilders(builders): - """Returns a list of slave dicts derived from the builders dict.""" - builders_in_pool = {} - - # builders.pyl contains a list of builders -> slave_pools - # and a list of slave_pools -> slaves. - # We require that each slave is in a single pool, but each slave - # may have multiple builders, so we need to build up the list of - # builders each slave pool supports. - for builder_name, builder_vals in builders['builders'].items(): - pool_names = builder_vals['slave_pools'] - for pool_name in pool_names: - if pool_name not in builders_in_pool: - builders_in_pool[pool_name] = set() - pool_data = builders['slave_pools'][pool_name] - for slave in pool_data['slaves']: - builders_in_pool[pool_name].add(builder_name) - - # Now we can generate the list of slaves using the above lookup table. - slaves = [] - for pool_name, pool_data in builders['slave_pools'].items(): - slave_data = pool_data['slave_data'] - builder_names = sorted(builders_in_pool[pool_name]) - for slave in pool_data['slaves']: - slaves.append({ - 'hostname': slave, - 'builder_name': builder_names, - 'master': builders['master_classname'], - 'os': slave_data['os'], - 'version': slave_data['version'], - 'bits': slave_data['bits'], - }) - - return slaves diff --git a/infra/scripts/legacy/scripts/common/env.py b/infra/scripts/legacy/scripts/common/env.py deleted file mode 100755 index 3e497ff..0000000 --- a/infra/scripts/legacy/scripts/common/env.py +++ /dev/null @@ -1,439 +0,0 @@ -#!/usr/bin/env python -# Copyright 2015 The Chromium Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -"""Implements a standard mechanism for Chrome Infra Python environment setup. - -This library provides a central location to define Chrome Infra environment -setup. It also provides several faculties to install this environment. - -Within a cooperating script, the environment can be setup by importing this -module and running its 'Install' method: - - # Install Chrome-Infra environment (replaces 'sys.path'). - sys.path.insert(0, - os.path.join(os.path.dirname(__file__), os.pardir, ...)) - # (/path/to/build/scripts) - import common.env - common.env.Install() - -When attempting to export the Chrome Infra path to external scripts, this -script can be invoked as an executable with various subcommands to emit a valid -PYTHONPATH clause. - -In addition, this module has several functions to construct the path. - -The goal is to deploy this module universally among Chrome-Infra scripts, -BuildBot configurations, tool invocations, and tests to ensure that they all -execute with the same centrally-defined environment. -""" - -import argparse -import collections -import contextlib -import imp -import itertools -import os -import sys -import traceback - - -# Export for bootstrapping. -__all__ = [ - 'Install', - 'PythonPath', - ] - - -# Name of enviornment extension file to seek. -ENV_EXTENSION_NAME = 'environment.cfg.py' - -# Standard directories (based on this file's location in the <build> tree). -def path_if(*args): - if not all(args): - return None - path = os.path.abspath(os.path.join(*args)) - return (path) if os.path.exists(path) else (None) - -# The path to the <build> directory in which this script resides. -Build = path_if(os.path.dirname(__file__), os.pardir, os.pardir) -# The path to the <build_internal> directory. -BuildInternal = path_if(Build, os.pardir, 'build_internal') - - -def SetPythonPathEnv(value): - """Sets the system's PYTHONPATH environemnt variable. - - Args: - value (str): The value to use. If this is empty/None, the system's - PYTHONPATH will be cleared. - """ - # Since we can't assign None to the environment "dictionary", we have to - # either set or delete the key depending on the original value. - if value is not None: - os.environ['PYTHONPATH'] = str(value) - else: - os.environ.pop('PYTHONPATH', None) - - -def Install(**kwargs): - """Replaces the current 'sys.path' with a hermetic Chrome-Infra path. - - Args: - kwargs (dict): See GetInfraPythonPath arguments. - - Returns (PythonPath): The PythonPath object that was installed. - """ - infra_python_path = GetInfraPythonPath(**kwargs) - infra_python_path.Install() - return infra_python_path - - -def SplitPath(path): - """Returns (list): A list of path elements. - - Splits a path into path elements. For example (assuming '/' is the local - system path separator): - >>> print SplitPath('/a/b/c/d') - ['/', 'a', 'b', 'c', 'd'] - >>> print SplitPath('a/b/c') - ['a', 'b,' 'c'] - """ - parts = [] - while True: - path, component = os.path.split(path) - if not component: - if path: - parts.append(path) - break - parts.append(component) - parts.reverse() - return parts - - -def ExtendPath(base, root_dir): - """Returns (PythonPath): The extended python path. - - This method looks for the ENV_EXTENSION_NAME file within "root_dir". If - present, it will be loaded as a Python module and have its "Extend" method - called. - - If no extension is found, the base PythonPath will be returned. - - Args: - base (PythonPath): The base python path. - root_dir (str): The path to check for an extension. - """ - extension_path = os.path.join(root_dir, ENV_EXTENSION_NAME) - if not os.path.isfile(extension_path): - return base - with open(extension_path, 'r') as fd: - extension = fd.read() - extension_module = imp.new_module('env-extension') - - # Execute the enviornment extension. - try: - exec extension in extension_module.__dict__ - - extend_func = getattr(extension_module, 'Extend', None) - assert extend_func, ( - "The environment extension module is missing the 'Extend()' method.") - base = extend_func(base, root_dir) - if not isinstance(base, PythonPath): - raise TypeError("Extension module returned non-PythonPath object (%s)" % ( - type(base).__name__,)) - except Exception: - # Re-raise the exception, but include the configuration file name. - tb = traceback.format_exc() - raise RuntimeError("Environment extension [%s] raised exception: %s" % ( - extension_path, tb)) - return base - - -def IsSystemPythonPath(path): - """Returns (bool): If a python path is user-installed. - - Paths that are known to be user-installed paths can be ignored when setting - up a hermetic Python path environment to avoid user libraries that would not - be present in other environments falsely affecting code. - - This function can be updated as-needed to exclude other non-system paths - encountered on bots and in the wild. - """ - components = SplitPath(path) - for component in components: - if component in ('dist-packages', 'site-packages'): - return False - return True - - -class PythonPath(collections.Sequence): - """An immutable set of Python path elements. - - All paths represented in this structure are absolute. If a relative path - is passed into this structure, it will be converted to absolute based on - the current working directory (via os.path.abspath). - """ - - def __init__(self, components=None): - """Initializes a new PythonPath instance. - - Args: - components (list): A list of path component strings. - """ - seen = set() - self._components = [] - for component in (components or ()): - component = os.path.abspath(component) - assert isinstance(component, basestring), ( - "Path component '%s' is not a string (%s)" % ( - component, type(component).__name__)) - if component in seen: - continue - seen.add(component) - self._components.append(component) - - def __getitem__(self, value): - return self._components[value] - - def __len__(self): - return len(self._components) - - def __iadd__(self, other): - return self.Append(other) - - def __repr__(self): - return self.pathstr - - def __eq__(self, other): - assert isinstance(other, type(self)) - return self._components == other._components - - @classmethod - def Flatten(cls, *paths): - """Returns (list): A single-level list containing flattened path elements. - - >>> print PythonPath.Flatten('a', ['b', ['c', 'd']]) - ['a', 'b', 'c', 'd'] - """ - result = [] - for path in paths: - if not isinstance(path, basestring): - # Assume it's an iterable of paths. - result += cls.Flatten(*path) - else: - result.append(path) - return result - - @classmethod - def FromPaths(cls, *paths): - """Returns (PythonPath): A PythonPath instantiated from path elements. - - Args: - paths (tuple): A tuple of path elements or iterables containing path - elements (e.g., PythonPath instances). - """ - return cls(cls.Flatten(*paths)) - - @classmethod - def FromPathStr(cls, pathstr): - """Returns (PythonPath): A PythonPath instantiated from the path string. - - Args: - pathstr (str): An os.pathsep()-delimited path string. - """ - return cls(pathstr.split(os.pathsep)) - - @property - def pathstr(self): - """Returns (str): A path string for the instance's path elements.""" - return os.pathsep.join(self) - - def IsHermetic(self): - """Returns (bool): True if this instance contains only system paths.""" - return all(IsSystemPythonPath(p) for p in self) - - def GetHermetic(self): - """Returns (PythonPath): derivative PythonPath containing only system paths. - """ - return type(self).FromPaths(*(p for p in self if IsSystemPythonPath(p))) - - def Append(self, *paths): - """Returns (PythonPath): derivative PythonPath with paths added to the end. - - Args: - paths (tuple): A tuple of path elements to append to the current instance. - """ - return type(self)(itertools.chain(self, self.FromPaths(*paths))) - - def Override(self, *paths): - """Returns (PythonPath): derivative PythonPath with paths prepended. - - Args: - paths (tuple): A tuple of path elements to prepend to the current - instance. - """ - return self.FromPaths(*paths).Append(self) - - def Install(self): - """Overwrites Python runtime variables based on the current instance. - - Performs the following operations: - - Replaces sys.path with the current instance's path. - - Replaces os.environ['PYTHONPATH'] with the current instance's path - string. - """ - sys.path = list(self) - SetPythonPathEnv(self.pathstr) - - @contextlib.contextmanager - def Enter(self): - """Context manager wrapper for Install. - - On exit, the context manager will restore the original environment. - """ - orig_sys_path = sys.path[:] - orig_pythonpath = os.environ.get('PYTHONPATH') - - try: - self.Install() - yield - finally: - sys.path = orig_sys_path - SetPythonPathEnv(orig_pythonpath) - - -def GetSysPythonPath(hermetic=True): - """Returns (PythonPath): A path based on 'sys.path'. - - Args: - hermetic (bool): If True, prune any non-system path. - """ - path = PythonPath.FromPaths(*sys.path) - if hermetic: - path = path.GetHermetic() - return path - - -def GetEnvPythonPath(): - """Returns (PythonPath): A path based on the PYTHONPATH environment variable. - """ - pythonpath = os.environ.get('PYTHONPATH') - if not pythonpath: - return PythonPath.FromPaths() - return PythonPath.FromPathStr(pythonpath) - - -def GetMasterPythonPath(master_dir): - """Returns (PythonPath): A path including a BuildBot master's directory. - - Args: - master_dir (str): The BuildBot master root directory. - """ - return PythonPath.FromPaths(master_dir) - - -def GetBuildPythonPath(): - """Returns (PythonPath): The Chrome Infra build path.""" - build_path = PythonPath.FromPaths() - for extension_dir in ( - Build, - BuildInternal, - ): - if extension_dir: - build_path = ExtendPath(build_path, extension_dir) - return build_path - - -def GetInfraPythonPath(hermetic=True, master_dir=None): - """Returns (PythonPath): The full working Chrome Infra utility path. - - This path is consistent for master, slave, and tool usage. It includes (in - this order): - - Any environment PYTHONPATH overrides. - - If 'master_dir' is supplied, the master's python path component. - - The Chrome Infra build path. - - The system python path. - - Args: - hermetic (bool): True, prune any non-system path from the system path. - master_dir (str): If not None, include a master path component. - """ - path = GetEnvPythonPath() - if master_dir: - path += GetMasterPythonPath(master_dir) - path += GetBuildPythonPath() - path += GetSysPythonPath(hermetic=hermetic) - return path - - -def _InfraPathFromArgs(args): - """Returns (PythonPath): A PythonPath populated from command-line arguments. - - Args: - args (argparse.Namespace): The command-line arguments constructed by 'main'. - """ - return GetInfraPythonPath( - master_dir=args.master_dir, - ) - - -def _Command_Echo(args, path): - """Returns (int): Return code. - - Command function for the 'echo' subcommand. Outputs the path string for - 'path'. - - Args: - args (argparse.Namespace): The command-line arguments constructed by 'main'. - path (PythonPath): The python path to use. - """ - args.output.write(path.pathstr) - return 0 - - -def _Command_Print(args, path): - """Returns (int): Return code. - - Command function for the 'print' subcommand. Outputs each path component in - path on a separate line. - - Args: - args (argparse.Namespace): The command-line arguments constructed by 'main'. - path (PythonPath): The python path to use. - """ - for component in path: - print >>args.output, component - return 0 - - -def main(): - """Main execution function.""" - parser = argparse.ArgumentParser() - parser.add_argument('-M', '--master_dir', - help="Augment the path with the master's directory.") - parser.add_argument('-o', '--output', metavar='PATH', - type=argparse.FileType('w'), default='-', - help="File to output to (use '-' for STDOUT).") - - subparsers = parser.add_subparsers() - - # 'echo' - subparser = subparsers.add_parser('echo') - subparser.set_defaults(func=_Command_Echo) - - # 'print' - subparser = subparsers.add_parser('print') - subparser.set_defaults(func=_Command_Print) - - # Parse - args = parser.parse_args() - - # Execute our subcommand function, which will return the exit code. - path = _InfraPathFromArgs(args) - return args.func(args, path) - - -if __name__ == '__main__': - sys.exit(main()) diff --git a/infra/scripts/legacy/scripts/common/url_helper.py b/infra/scripts/legacy/scripts/common/url_helper.py deleted file mode 100644 index 78adc86..0000000 --- a/infra/scripts/legacy/scripts/common/url_helper.py +++ /dev/null @@ -1,60 +0,0 @@ -# Copyright 2013 The Chromium Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -import mimetypes -import urllib2 - - -def GetMimeType(filename): - return mimetypes.guess_type(filename)[0] or 'application/octet-stream' - - -def EncodeMultipartFormData(fields, files): - """Encode form fields for multipart/form-data. - - Args: - fields: A sequence of (name, value) elements for regular form fields. - files: A sequence of (name, filename, value) elements for data to be - uploaded as files. - Returns: - (content_type, body) ready for httplib.HTTP instance. - - Source: - http://code.google.com/p/rietveld/source/browse/trunk/upload.py - """ - BOUNDARY = '-M-A-G-I-C---B-O-U-N-D-A-R-Y-' - CRLF = '\r\n' - lines = [] - - for key, value in fields: - lines.append('--' + BOUNDARY) - lines.append('Content-Disposition: form-data; name="%s"' % key) - lines.append('') - if isinstance(value, unicode): - value = value.encode('utf-8') - lines.append(value) - - for key, filename, value in files: - lines.append('--' + BOUNDARY) - lines.append('Content-Disposition: form-data; name="%s"; filename="%s"' % - (key, filename)) - lines.append('Content-Type: %s' % GetMimeType(filename)) - lines.append('') - if isinstance(value, unicode): - value = value.encode('utf-8') - lines.append(value) - - lines.append('--' + BOUNDARY + '--') - lines.append('') - body = CRLF.join(lines) - content_type = 'multipart/form-data; boundary=%s' % BOUNDARY - return content_type, body - - -def upload_files(url, attrs, file_objs): - content_type, data = EncodeMultipartFormData(attrs, file_objs) - headers = {"Content-Type": content_type} - request = urllib2.Request(url, data, headers) - - return urllib2.urlopen(request) diff --git a/infra/scripts/legacy/scripts/slave/bootstrap.py b/infra/scripts/legacy/scripts/slave/bootstrap.py deleted file mode 100644 index 2bfa4ce..0000000 --- a/infra/scripts/legacy/scripts/slave/bootstrap.py +++ /dev/null @@ -1,47 +0,0 @@ -# Copyright (c) 2012 The Chromium Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -"""Utilities to enable slaves to determine their master without importing any -buildbot or twisted code. -""" - -import inspect -import os -import sys - -from common import chromium_utils -import config_bootstrap - - -def ImportMasterConfigs(master_name=None, include_internal=True): - """Imports master configs. - - Normally a slave can use chromium_utils.GetActiveMaster() to find - itself and determine which ActiveMaster to use. In that case, the - active master name is passed in as an arg, and we only load the - site_config.py that defines it. When testing, the current "slave" - won't be found. In that case, we don't know which config to use, so - load them all. In either case, masters are assigned as attributes - to the config.Master object. - """ - for master in chromium_utils.ListMasters(include_internal=include_internal): - path = os.path.join(master, 'master_site_config.py') - if os.path.exists(path): - local_vars = {} - try: - execfile(path, local_vars) - # pylint: disable=W0703 - except Exception, e: - # Naked exceptions are banned by the style guide but we are - # trying to be resilient here. - print >> sys.stderr, 'WARNING: cannot exec ' + path - print >> sys.stderr, e - for (symbol_name, symbol) in local_vars.iteritems(): - if inspect.isclass(symbol): - setattr(symbol, 'local_config_path', master) - setattr(config_bootstrap.Master, symbol_name, symbol) - # If we have a master_name and it matches, set - # config_bootstrap.Master.active_master. - if master_name and master_name == symbol_name: - setattr(config_bootstrap.Master, 'active_master', symbol) diff --git a/infra/scripts/legacy/scripts/slave/gtest/__init__.py b/infra/scripts/legacy/scripts/slave/gtest/__init__.py deleted file mode 100644 index e69de29..0000000 --- a/infra/scripts/legacy/scripts/slave/gtest/__init__.py +++ /dev/null diff --git a/infra/scripts/legacy/scripts/slave/gtest/json_results_generator.py b/infra/scripts/legacy/scripts/slave/gtest/json_results_generator.py deleted file mode 100644 index cf207b0..0000000 --- a/infra/scripts/legacy/scripts/slave/gtest/json_results_generator.py +++ /dev/null @@ -1,255 +0,0 @@ -# Copyright (c) 2012 The Chromium Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -"""A utility class to generate JSON results from given test results and upload -them to the specified results server. - -""" - -from __future__ import with_statement - -import codecs -import logging -import os -import time - -import simplejson -from slave.gtest.test_result import TestResult -from slave.gtest.test_results_uploader import TestResultsUploader - -# A JSON results generator for generic tests. - -JSON_PREFIX = 'ADD_RESULTS(' -JSON_SUFFIX = ');' - - -def test_did_pass(test_result): - return not test_result.failed and test_result.modifier == TestResult.NONE - - -def add_path_to_trie(path, value, trie): - """Inserts a single flat directory path and associated value into a directory - trie structure.""" - if not '/' in path: - trie[path] = value - return - - # we don't use slash - # pylint: disable=W0612 - directory, slash, rest = path.partition('/') - if not directory in trie: - trie[directory] = {} - add_path_to_trie(rest, value, trie[directory]) - - -def generate_test_timings_trie(individual_test_timings): - """Breaks a test name into chunks by directory and puts the test time as a - value in the lowest part, e.g. - foo/bar/baz.html: 1ms - foo/bar/baz1.html: 3ms - - becomes - foo: { - bar: { - baz.html: 1, - baz1.html: 3 - } - } - """ - trie = {} - # Only use the timing of the first try of each test. - for test_results in individual_test_timings: - test = test_results[0].test_name - - add_path_to_trie(test, int(1000 * test_results[-1].test_run_time), trie) - - return trie - - -class JSONResultsGenerator(object): - """A JSON results generator for generic tests.""" - - FAIL_LABEL = 'FAIL' - PASS_LABEL = 'PASS' - FLAKY_LABEL = ' '.join([FAIL_LABEL, PASS_LABEL]) - SKIP_LABEL = 'SKIP' - - ACTUAL = 'actual' - BLINK_REVISION = 'blink_revision' - BUILD_NUMBER = 'build_number' - BUILDER_NAME = 'builder_name' - CHROMIUM_REVISION = 'chromium_revision' - EXPECTED = 'expected' - FAILURE_SUMMARY = 'num_failures_by_type' - SECONDS_SINCE_EPOCH = 'seconds_since_epoch' - TEST_TIME = 'time' - TESTS = 'tests' - VERSION = 'version' - VERSION_NUMBER = 3 - - RESULTS_FILENAME = 'results.json' - TIMES_MS_FILENAME = 'times_ms.json' - FULL_RESULTS_FILENAME = 'full_results.json' - - def __init__(self, builder_name, build_name, build_number, - results_file_base_path, builder_base_url, - test_results_map, svn_revisions=None, - test_results_server=None, - test_type='', - master_name='', - file_writer=None): - """Modifies the results.json file. Grabs it off the archive directory - if it is not found locally. - - Args - builder_name: the builder name (e.g. Webkit). - build_name: the build name (e.g. webkit-rel). - build_number: the build number. - results_file_base_path: Absolute path to the directory containing the - results json file. - builder_base_url: the URL where we have the archived test results. - If this is None no archived results will be retrieved. - test_results_map: A dictionary that maps test_name to a list of - TestResult, one for each time the test was retried. - svn_revisions: A (json_field_name, revision) pair for SVN - repositories that tests rely on. The SVN revision will be - included in the JSON with the given json_field_name. - test_results_server: server that hosts test results json. - test_type: test type string (e.g. 'layout-tests'). - master_name: the name of the buildbot master. - file_writer: if given the parameter is used to write JSON data to a file. - The parameter must be the function that takes two arguments, 'file_path' - and 'data' to be written into the file_path. - """ - self._builder_name = builder_name - self._build_name = build_name - self._build_number = build_number - self._builder_base_url = builder_base_url - self._results_directory = results_file_base_path - - self._test_results_map = test_results_map - - self._svn_revisions = svn_revisions - if not self._svn_revisions: - self._svn_revisions = {} - - self._test_results_server = test_results_server - self._test_type = test_type - self._master_name = master_name - self._file_writer = file_writer - - def generate_json_output(self): - json = self.get_full_results_json() - if json: - file_path = os.path.join(self._results_directory, - self.FULL_RESULTS_FILENAME) - self._write_json(json, file_path) - - def generate_times_ms_file(self): - times = generate_test_timings_trie(self._test_results_map.values()) - file_path = os.path.join(self._results_directory, self.TIMES_MS_FILENAME) - self._write_json(times, file_path) - - def get_full_results_json(self): - results = {self.VERSION: self.VERSION_NUMBER} - - # Metadata generic to all results. - results[self.BUILDER_NAME] = self._builder_name - results[self.BUILD_NUMBER] = self._build_number - results[self.SECONDS_SINCE_EPOCH] = int(time.time()) - for name, revision in self._svn_revisions: - results[name + '_revision'] = revision - - tests = results.setdefault(self.TESTS, {}) - for test_name in self._test_results_map.iterkeys(): - tests[test_name] = self._make_test_data(test_name) - - self._insert_failure_map(results) - - return results - - def _insert_failure_map(self, results): - # FAIL, PASS, NOTRUN - summary = {self.PASS_LABEL: 0, self.FAIL_LABEL: 0, self.SKIP_LABEL: 0} - for test_results in self._test_results_map.itervalues(): - # Use the result of the first test for aggregate statistics. This may - # count as failing a test that passed on retry, but it's a more useful - # statistic and it's consistent with our other test harnesses. - test_result = test_results[0] - if test_did_pass(test_result): - summary[self.PASS_LABEL] += 1 - elif test_result.modifier == TestResult.DISABLED: - summary[self.SKIP_LABEL] += 1 - elif test_result.failed: - summary[self.FAIL_LABEL] += 1 - - results[self.FAILURE_SUMMARY] = summary - - def _make_test_data(self, test_name): - test_data = {} - expected, actual = self._get_expected_and_actual_results(test_name) - test_data[self.EXPECTED] = expected - test_data[self.ACTUAL] = actual - # Use the timing of the first try, it's a better representative since it - # runs under more load than retries. - run_time = int(self._test_results_map[test_name][0].test_run_time) - test_data[self.TEST_TIME] = run_time - - return test_data - - def _get_expected_and_actual_results(self, test_name): - test_results = self._test_results_map[test_name] - # Use the modifier of the first try, they should all be the same. - modifier = test_results[0].modifier - - if modifier == TestResult.DISABLED: - return (self.SKIP_LABEL, self.SKIP_LABEL) - - actual_list = [] - for test_result in test_results: - label = self.FAIL_LABEL if test_result.failed else self.PASS_LABEL - actual_list.append(label) - actual = " ".join(actual_list) - - if modifier == TestResult.NONE: - return (self.PASS_LABEL, actual) - if modifier == TestResult.FLAKY: - return (self.FLAKY_LABEL, actual) - if modifier == TestResult.FAILS: - return (self.FAIL_LABEL, actual) - - def upload_json_files(self, json_files): - """Uploads the given json_files to the test_results_server (if the - test_results_server is given).""" - if not self._test_results_server: - return - - if not self._master_name: - logging.error('--test-results-server was set, but --master-name was not. ' - 'Not uploading JSON files.') - return - - print 'Uploading JSON files for builder: %s' % self._builder_name - attrs = [('builder', self._builder_name), - ('testtype', self._test_type), - ('master', self._master_name)] - - files = [(f, os.path.join(self._results_directory, f)) for f in json_files] - - uploader = TestResultsUploader(self._test_results_server) - # Set uploading timeout in case appengine server is having problem. - # 120 seconds are more than enough to upload test results. - uploader.upload(attrs, files, 120) - - print 'JSON files uploaded.' - - def _write_json(self, json_object, file_path): - # Specify separators in order to get compact encoding. - json_data = simplejson.dumps(json_object, separators=(',', ':')) - json_string = json_data - if self._file_writer: - self._file_writer(file_path, json_string) - else: - with codecs.open(file_path, 'w', 'utf8') as f: - f.write(json_string) diff --git a/infra/scripts/legacy/scripts/slave/gtest/networktransaction.py b/infra/scripts/legacy/scripts/slave/gtest/networktransaction.py deleted file mode 100644 index 10327cb..0000000 --- a/infra/scripts/legacy/scripts/slave/gtest/networktransaction.py +++ /dev/null @@ -1,46 +0,0 @@ -# Copyright (c) 2011 The Chromium Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -import logging -import time -import urllib2 - - -class NetworkTimeout(Exception): - pass - - -class NetworkTransaction(object): - def __init__(self, initial_backoff_seconds=10, grown_factor=1.5, - timeout_seconds=(10 * 60), convert_404_to_None=False): - self._initial_backoff_seconds = initial_backoff_seconds - self._backoff_seconds = initial_backoff_seconds - self._grown_factor = grown_factor - self._timeout_seconds = timeout_seconds - self._convert_404_to_None = convert_404_to_None - self._total_sleep = 0 - - def run(self, request): - self._total_sleep = 0 - self._backoff_seconds = self._initial_backoff_seconds - while True: - try: - return request() - except urllib2.HTTPError, e: - if self._convert_404_to_None and e.code == 404: - return None - self._check_for_timeout() - logging.warn("Received HTTP status %s loading \"%s\". " - "Retrying in %s seconds...", - e.code, e.filename, self._backoff_seconds) - self._sleep() - - def _check_for_timeout(self): - if self._total_sleep + self._backoff_seconds > self._timeout_seconds: - raise NetworkTimeout() - - def _sleep(self): - time.sleep(self._backoff_seconds) - self._total_sleep += self._backoff_seconds - self._backoff_seconds *= self._grown_factor diff --git a/infra/scripts/legacy/scripts/slave/gtest/test_result.py b/infra/scripts/legacy/scripts/slave/gtest/test_result.py deleted file mode 100644 index 3efd37d..0000000 --- a/infra/scripts/legacy/scripts/slave/gtest/test_result.py +++ /dev/null @@ -1,42 +0,0 @@ -# Copyright (c) 2012 The Chromium Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - - -def canonical_name(name): - new_name = name.replace('FLAKY_', '', 1) - new_name = new_name.replace('FAILS_', '', 1) - new_name = new_name.replace('DISABLED_', '', 1) - return new_name - - -class TestResult(object): - """A simple class that represents a single test result.""" - - # Test modifier constants. - (NONE, FAILS, FLAKY, DISABLED) = range(4) - - def __init__(self, test, failed, not_run=False, elapsed_time=0): - self.test_name = canonical_name(test) - self.failed = failed - self.test_run_time = elapsed_time - - test_name = test - try: - test_name = test.split('.')[1] - except IndexError: - pass - - if not_run: - self.modifier = self.DISABLED - elif test_name.startswith('FAILS_'): - self.modifier = self.FAILS - elif test_name.startswith('FLAKY_'): - self.modifier = self.FLAKY - elif test_name.startswith('DISABLED_'): - self.modifier = self.DISABLED - else: - self.modifier = self.NONE - - def fixable(self): - return self.failed or self.modifier == self.DISABLED diff --git a/infra/scripts/legacy/scripts/slave/gtest/test_results_uploader.py b/infra/scripts/legacy/scripts/slave/gtest/test_results_uploader.py deleted file mode 100644 index ead70fb..0000000 --- a/infra/scripts/legacy/scripts/slave/gtest/test_results_uploader.py +++ /dev/null @@ -1,32 +0,0 @@ -# Copyright (c) 2011 The Chromium Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -import codecs -import socket - -from common import url_helper -from slave.gtest.networktransaction import NetworkTransaction - - -class TestResultsUploader(object): - def __init__(self, host): - self._host = host - - def _upload_files(self, attrs, file_objs): - url = "http://%s/testfile/upload" % self._host - url_helper.upload_files(url, attrs, file_objs) - - def upload(self, params, files, timeout_seconds): - file_objs = [] - for filename, path in files: - with codecs.open(path, "rb") as f: - file_objs.append(('file', filename, f.read())) - - orig_timeout = socket.getdefaulttimeout() - try: - socket.setdefaulttimeout(timeout_seconds) - NetworkTransaction(timeout_seconds=timeout_seconds).run( - lambda: self._upload_files(params, file_objs)) - finally: - socket.setdefaulttimeout(orig_timeout) diff --git a/infra/scripts/legacy/scripts/slave/gtest_slave_utils.py b/infra/scripts/legacy/scripts/slave/gtest_slave_utils.py deleted file mode 100755 index 2b4135c..0000000 --- a/infra/scripts/legacy/scripts/slave/gtest_slave_utils.py +++ /dev/null @@ -1,103 +0,0 @@ -#!/usr/bin/env python -# Copyright (c) 2012 The Chromium Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -import logging -import os - -from common import gtest_utils -from slave.gtest.json_results_generator import JSONResultsGenerator -from slave.gtest.test_result import canonical_name -from slave.gtest.test_result import TestResult - - -GENERATE_JSON_RESULTS_OPTIONS = [ - 'builder_name', 'build_name', 'build_number', 'results_directory', - 'builder_base_url', 'webkit_revision', 'chrome_revision', - 'test_results_server', 'test_type', 'master_name'] - -FULL_RESULTS_FILENAME = 'full_results.json' -TIMES_MS_FILENAME = 'times_ms.json' - - -def GetResultsMap(observer): - """Returns a map of TestResults.""" - - test_results_map = dict() - tests = (observer.FailedTests(include_fails=True, include_flaky=True) + - observer.PassedTests()) - for test in tests: - key = canonical_name(test) - test_results_map[key] = [] - tries = observer.TriesForTest(test) - for test_try in tries: - # FIXME: Store the actual failure type so we can expose whether the test - # crashed or timed out. See crbug.com/249965. - failed = (test_try != gtest_utils.TEST_SUCCESS_LABEL) - test_results_map[key].append(TestResult(test, failed=failed)) - - return test_results_map - - -def GenerateJSONResults(test_results_map, options): - """Generates a JSON results file from the given test_results_map, - returning the associated generator for use with UploadJSONResults, below. - - Args: - test_results_map: A map of TestResult. - options: options for json generation. See GENERATE_JSON_RESULTS_OPTIONS - and OptionParser's help messages below for expected options and their - details. - """ - - if not test_results_map: - logging.warn('No input results map was given.') - return - - # Make sure we have all the required options (set empty string otherwise). - for opt in GENERATE_JSON_RESULTS_OPTIONS: - if not getattr(options, opt, None): - logging.warn('No value is given for option %s', opt) - setattr(options, opt, '') - - try: - int(options.build_number) - except ValueError: - logging.error('options.build_number needs to be a number: %s', - options.build_number) - return - - if not os.path.exists(options.results_directory): - os.makedirs(options.results_directory) - - print('Generating json: ' - 'builder_name:%s, build_name:%s, build_number:%s, ' - 'results_directory:%s, builder_base_url:%s, ' - 'webkit_revision:%s, chrome_revision:%s ' - 'test_results_server:%s, test_type:%s, master_name:%s' % - (options.builder_name, options.build_name, options.build_number, - options.results_directory, options.builder_base_url, - options.webkit_revision, options.chrome_revision, - options.test_results_server, options.test_type, - options.master_name)) - - generator = JSONResultsGenerator( - options.builder_name, options.build_name, options.build_number, - options.results_directory, options.builder_base_url, - test_results_map, - svn_revisions=(('blink', options.webkit_revision), - ('chromium', options.chrome_revision)), - test_results_server=options.test_results_server, - test_type=options.test_type, - master_name=options.master_name) - generator.generate_json_output() - generator.generate_times_ms_file() - return generator - -def UploadJSONResults(generator): - """Conditionally uploads the results from GenerateJSONResults if - test_results_server was given.""" - if generator: - generator.upload_json_files([FULL_RESULTS_FILENAME, - TIMES_MS_FILENAME]) diff --git a/infra/scripts/legacy/scripts/slave/runtest.py b/infra/scripts/legacy/scripts/slave/runtest.py index 42ee7db..3e09630 100755 --- a/infra/scripts/legacy/scripts/slave/runtest.py +++ b/infra/scripts/legacy/scripts/slave/runtest.py @@ -23,12 +23,8 @@ import sys from common import chromium_utils from common import gtest_utils -# TODO(crbug.com/403564). We almost certainly shouldn't be importing this. -import config - from slave import annotation_utils from slave import build_directory -from slave import gtest_slave_utils from slave import slave_utils from slave import xvfb @@ -36,9 +32,6 @@ USAGE = '%s [options] test.exe [test args]' % os.path.basename(sys.argv[0]) CHROME_SANDBOX_PATH = '/opt/chromium/chrome_sandbox' -# Directory to write JSON for test results into. -DEST_DIR = 'gtest_results' - # The directory that this script is in. BASE_DIR = os.path.dirname(os.path.abspath(__file__)) @@ -131,95 +124,6 @@ def _RunGTestCommand( return chromium_utils.RunCommand(command, pipes=pipes, env=env) -def _GetMaster(): - """Return the master name for the current host.""" - return chromium_utils.GetActiveMaster() - - -def _GetMasterString(master): - """Returns a message describing what the master is.""" - return '[Running for master: "%s"]' % master - - -def _GenerateJSONForTestResults(options, log_processor): - """Generates or updates a JSON file from the gtest results XML and upload the - file to the archive server. - - The archived JSON file will be placed at: - www-dir/DEST_DIR/buildname/testname/results.json - on the archive server. NOTE: This will be deprecated. - - Args: - options: command-line options that are supposed to have build_dir, - results_directory, builder_name, build_name and test_output_xml values. - log_processor: An instance of PerformanceLogProcessor or similar class. - - Returns: - True upon success, False upon failure. - """ - results_map = None - try: - results_map = gtest_slave_utils.GetResultsMap(log_processor) - except Exception as e: - # This error will be caught by the following 'not results_map' statement. - print 'Error: ', e - - if not results_map: - print 'No data was available to update the JSON results' - # Consider this non-fatal. - return True - - build_dir = os.path.abspath(options.build_dir) - slave_name = options.builder_name or slave_utils.SlaveBuildName(build_dir) - - generate_json_options = copy.copy(options) - generate_json_options.build_name = slave_name - generate_json_options.input_results_xml = options.test_output_xml - generate_json_options.builder_base_url = '%s/%s/%s/%s' % ( - config.Master.archive_url, DEST_DIR, slave_name, options.test_type) - generate_json_options.master_name = options.master_class_name or _GetMaster() - generate_json_options.test_results_server = config.Master.test_results_server - - print _GetMasterString(generate_json_options.master_name) - - generator = None - - try: - if options.revision: - generate_json_options.chrome_revision = options.revision - else: - generate_json_options.chrome_revision = '' - - if options.webkit_revision: - generate_json_options.webkit_revision = options.webkit_revision - else: - generate_json_options.webkit_revision = '' - - # Generate results JSON file and upload it to the appspot server. - generator = gtest_slave_utils.GenerateJSONResults( - results_map, generate_json_options) - - except Exception as e: - print 'Unexpected error while generating JSON: %s' % e - sys.excepthook(*sys.exc_info()) - return False - - # The code can throw all sorts of exceptions, including - # slave.gtest.networktransaction.NetworkTimeout so just trap everything. - # Earlier versions of this code ignored network errors, so until a - # retry mechanism is added, continue to do so rather than reporting - # an error. - try: - # Upload results JSON file to the appspot server. - gtest_slave_utils.UploadJSONResults(generator) - except Exception as e: - # Consider this non-fatal for the moment. - print 'Unexpected error while uploading JSON: %s' % e - sys.excepthook(*sys.exc_info()) - - return True - - def _BuildTestBinaryCommand(_build_dir, test_exe_path, options): """Builds a command to run a test binary. @@ -381,11 +285,6 @@ def _Main(options, args, extra_env): log_processor = gtest_utils.GTestJSONParser( options.build_properties.get('mastername')) - if options.generate_json_file: - if os.path.exists(options.test_output_xml): - # remove the old XML output file. - os.remove(options.test_output_xml) - try: # TODO(dpranke): checking on test_exe is a temporary hack until we # can change the buildbot master to pass --xvfb instead of --no-xvfb @@ -426,10 +325,6 @@ def _Main(options, args, extra_env): _SymbolizeSnippetsInJSON(options, json_file_name) log_processor.ProcessJSONFile(options.build_dir) - if options.generate_json_file: - if not _GenerateJSONForTestResults(options, log_processor): - return 1 - if options.annotate: annotation_utils.annotate(options.test_type, result, log_processor) @@ -708,15 +603,6 @@ def main(): extra_env['GTEST_TOTAL_SHARDS'] = str(options.total_shards) extra_env['GTEST_SHARD_INDEX'] = str(options.shard_index - 1) - if options.results_directory: - options.test_output_xml = os.path.normpath(os.path.abspath(os.path.join( - options.results_directory, '%s.xml' % options.test_type))) - args.append('--gtest_output=xml:' + options.test_output_xml) - elif options.generate_json_file: - option_parser.error( - '--results-directory is required with --generate-json-file=True') - return 1 - return _Main(options, args, extra_env) finally: if did_launch_dbus: diff --git a/infra/scripts/legacy/site_config/config.py b/infra/scripts/legacy/site_config/config.py deleted file mode 100644 index 0108e65..0000000 --- a/infra/scripts/legacy/site_config/config.py +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright (c) 2012 The Chromium Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -"""Declares a number of site-dependent variables for use by scripts. -""" - -import os - -# DatabaseSetup was moved. Import it for backward compatibility -from common.chromium_utils import DatabaseSetup # pylint: disable=W0611 -from config_bootstrap import config_private # pylint: disable=W0403,W0611 -from config_bootstrap import Master # pylint: disable=W0403,W0611 - -SITE_CONFIG_PATH = os.path.abspath(os.path.dirname(__file__)) - -def SiteConfigPath(): - return SITE_CONFIG_PATH diff --git a/infra/scripts/legacy/site_config/config_bootstrap.py b/infra/scripts/legacy/site_config/config_bootstrap.py deleted file mode 100644 index 8777cab..0000000 --- a/infra/scripts/legacy/site_config/config_bootstrap.py +++ /dev/null @@ -1,128 +0,0 @@ -# Copyright (c) 2012 The Chromium Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -""" -Site configuration information that is sufficient to configure a slave, -without loading any buildbot or twisted code. -""" - -import inspect -import os - -BASE_DIR = os.path.dirname(os.path.abspath(__file__)) - -# Override config_default with a config_private file. -BASE_MASTERS = [] -try: - import config_private # pylint: disable=F0401 - BASE_MASTERS += [config_private.Master, config_private.PublicMaster] -except ImportError: - import config_default as config_private # pylint: disable=W0403 - BASE_MASTERS += [config_private.Master,] - - -class Master(config_private.Master): - """Buildbot master configuration options.""" - - trunk_url = (config_private.Master.server_url + - config_private.Master.repo_root + '/trunk') - - webkit_trunk_url = (config_private.Master.webkit_root_url + '/trunk') - - trunk_url_src = config_private.Master.git_server_url + '/chromium/src.git' - trunk_url_tools = trunk_url + '/tools' - nacl_url = config_private.Master.nacl_trunk_url + '/src/native_client' - nacl_sdk_root_url = 'https://nativeclient-sdk.googlecode.com/svn' - nacl_ports_trunk_url = 'https://naclports.googlecode.com/svn/trunk' - nacl_ports_url = nacl_ports_trunk_url + '/src' - gears_url = 'http://gears.googlecode.com/svn/trunk' - gyp_trunk_url = 'http://gyp.googlecode.com/svn/trunk' - branch_url = (config_private.Master.server_url + - config_private.Master.repo_root + '/branches') - merge_branch_url = branch_url + '/chrome_webkit_merge_branch' - merge_branch_url_src = merge_branch_url + '/src' - - v8_url = 'http://v8.googlecode.com/svn' - v8_branch_url = (v8_url + '/branches') - v8_bleeding_edge = v8_branch_url + '/bleeding_edge' - v8_trunk = v8_url + '/trunk' - es5conform_root_url = "https://es5conform.svn.codeplex.com/svn/" - es5conform_revision = 62998 - - dart_url = config_private.Master.googlecode_url % 'dart' - dart_bleeding = dart_url + '/branches/bleeding_edge' - dart_trunk = dart_url + '/trunk' - - oilpan_url = (config_private.Master.webkit_root_url + '/branches/oilpan') - - skia_url = 'http://skia.googlecode.com/svn/' - - syzygy_url = 'http://sawbuck.googlecode.com/svn/' - - webrtc_url = config_private.Master.googlecode_url % 'webrtc' - libyuv_url = 'http://libyuv.googlecode.com/svn' - - # Default target platform if none was given to the factory. - default_platform = 'win32' - - # Used by the waterfall display. - project_url = 'http://www.chromium.org' - - # Base URL for perf test results. - perf_base_url = 'http://build.chromium.org/f/chromium/perf' - - # Suffix for perf URL. - perf_report_url_suffix = 'report.html?history=150' - - # Directory in which to save perf-test output data files. - perf_output_dir = '~/www/perf' - - # URL pointing to builds and test results. - archive_url = 'http://build.chromium.org/buildbot' - - # The test results server to upload our test results. - test_results_server = 'test-results.appspot.com' - - # File in which to save a list of graph names. - perf_graph_list = 'graphs.dat' - - # Magic step return code inidicating "warning(s)" rather than "error". - retcode_warnings = 88 - - @staticmethod - def GetBotPassword(): - """Returns the slave password retrieved from a local file, or None. - - The slave password is loaded from a local file next to this module file, if - it exists. This is a function rather than a variable so it's not called - when it's not needed. - - We can't both make this a property and also keep it static unless we use a - <metaclass, which is overkill for this usage. - """ - # Note: could be overriden by config_private. - if not getattr(Master, 'bot_password', None): - # If the bot_password has been requested, the file is required to exist - # if not overriden in config_private. - bot_password_path = os.path.join(BASE_DIR, '.bot_password') - Master.bot_password = open(bot_password_path).read().strip('\n\r') - return Master.bot_password - - @staticmethod - def _extract_masters(master): - return [v for v in master.__dict__.itervalues() - if (inspect.isclass(v) and - issubclass(v, config_private.Master.Base) and - v != config_private.Master.Base)] - - @classmethod - def get_base_masters(cls): - masters = [] - for base_master in BASE_MASTERS: - masters += cls._extract_masters(base_master) - return masters - - @classmethod - def get_all_masters(cls): - return cls._extract_masters(cls) diff --git a/infra/scripts/legacy/site_config/config_default.py b/infra/scripts/legacy/site_config/config_default.py deleted file mode 100644 index 8d0cb0a..0000000 --- a/infra/scripts/legacy/site_config/config_default.py +++ /dev/null @@ -1,230 +0,0 @@ -# Copyright 2012 The Chromium Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -"""Seeds a number of variables defined in chromium_config.py. - -The recommended way is to fork this file and use a custom DEPS forked from -config/XXX/DEPS with the right configuration data.""" - -import os -import re -import socket - - -SERVICE_ACCOUNTS_PATH = '/creds/service_accounts' - - -class classproperty(object): - """A decorator that allows is_production_host to only to be defined once.""" - def __init__(self, getter): - self.getter = getter - def __get__(self, instance, owner): - return self.getter(owner) - - -class Master(object): - # Repository URLs used by the SVNPoller and 'gclient config'. - server_url = 'http://src.chromium.org' - repo_root = '/svn' - git_server_url = 'https://chromium.googlesource.com' - - # External repos. - googlecode_url = 'http://%s.googlecode.com/svn' - sourceforge_url = 'https://svn.code.sf.net/p/%(repo)s/code' - googlecode_revlinktmpl = 'https://code.google.com/p/%s/source/browse?r=%s' - - # Directly fetches from anonymous Blink svn server. - webkit_root_url = 'http://src.chromium.org/blink' - nacl_trunk_url = 'http://src.chromium.org/native_client/trunk' - - llvm_url = 'http://llvm.org/svn/llvm-project' - - # Perf Dashboard upload URL. - dashboard_upload_url = 'https://chromeperf.appspot.com' - - # Actually for Chromium OS slaves. - chromeos_url = git_server_url + '/chromiumos.git' - - # Default domain for emails to come from and - # domains to which emails can be sent. - master_domain = 'example.com' - permitted_domains = ('example.com',) - - # Your smtp server to enable mail notifications. - smtp = 'smtp' - - # By default, bot_password will be filled in by config.GetBotPassword(). - bot_password = None - - # Fake urls to make various factories happy. - trunk_internal_url = None - trunk_internal_url_src = None - slave_internal_url = None - git_internal_server_url = None - syzygy_internal_url = None - v8_internal_url = None - - - class Base(object): - """Master base template. - Contains stubs for variables that all masters must define.""" - - # Base service offset for 'master_port' - MASTER_PORT = 2 - # Base service offset for 'slave_port' - SLAVE_PORT = 3 - # Base service offset for 'master_port_alt' - MASTER_PORT_ALT = 4 - # Base service offset for 'try_job_port' - TRY_JOB_PORT = 5 - - # A BuildBucket bucket to poll. - buildbucket_bucket = None - - # Master address. You should probably copy this file in another svn repo - # so you can override this value on both the slaves and the master. - master_host = 'localhost' - @classproperty - def current_host(cls): - return socket.getfqdn() - @classproperty - def in_production(cls): - return re.match(r'master.*\.golo\.chromium\.org', cls.current_host) - # Only report that we are running on a master if the master_host (even when - # master_host is overridden by a subclass) is the same as the current host. - @classproperty - def is_production_host(cls): - return cls.current_host == cls.master_host - - # 'from:' field for emails sent from the server. - from_address = 'nobody@example.com' - # Additional email addresses to send gatekeeper (automatic tree closage) - # notifications. Unnecessary for experimental masters and try servers. - tree_closing_notification_recipients = [] - - @classproperty - def master_port(cls): - return cls._compose_port(cls.MASTER_PORT) - - @classproperty - def slave_port(cls): - # Which port slaves use to connect to the master. - return cls._compose_port(cls.SLAVE_PORT) - - @classproperty - def master_port_alt(cls): - # The alternate read-only page. Optional. - return cls._compose_port(cls.MASTER_PORT_ALT) - - @classproperty - def try_job_port(cls): - return cls._compose_port(cls.TRY_JOB_PORT) - - @classmethod - def _compose_port(cls, service): - """Returns: The port number for 'service' from the master's static config. - - Port numbers are mapped of the form: - XYYZZ - || \__The last two digits identify the master, e.g. master.chromium - |\____The second and third digits identify the master host, e.g. - | master1.golo - \_____The first digit identifies the port type, e.g. master_port - - If any configuration is missing (incremental migration), this method will - return '0' for that query, indicating no port. - """ - return ( - (service * 10000) + # X - (cls.master_port_base * 100) + # YY - cls.master_port_id) # ZZ - - service_account_file = None - - @classproperty - def service_account_path(cls): - if cls.service_account_file is None: - return None - return os.path.join(SERVICE_ACCOUNTS_PATH, cls.service_account_file) - - ## Per-master configs. - - class Master1(Base): - """Chromium master.""" - master_host = 'master1.golo.chromium.org' - master_port_base = 1 - from_address = 'buildbot@chromium.org' - tree_closing_notification_recipients = [ - 'chromium-build-failure@chromium-gatekeeper-sentry.appspotmail.com'] - base_app_url = 'https://chromium-status.appspot.com' - tree_status_url = base_app_url + '/status' - store_revisions_url = base_app_url + '/revisions' - last_good_url = base_app_url + '/lkgr' - last_good_blink_url = 'http://blink-status.appspot.com/lkgr' - - class Master2(Base): - """Legacy ChromeOS master.""" - master_host = 'master2.golo.chromium.org' - master_port_base = 2 - tree_closing_notification_recipients = [ - 'chromeos-build-failures@google.com'] - from_address = 'buildbot@chromium.org' - - class Master2a(Base): - """Chromeos master.""" - master_host = 'master2a.golo.chromium.org' - master_port_base = 15 - tree_closing_notification_recipients = [ - 'chromeos-build-failures@google.com'] - from_address = 'buildbot@chromium.org' - - class Master3(Base): - """Client master.""" - master_host = 'master3.golo.chromium.org' - master_port_base = 3 - tree_closing_notification_recipients = [] - from_address = 'buildbot@chromium.org' - - class Master4(Base): - """Try server master.""" - master_host = 'master4.golo.chromium.org' - master_port_base = 4 - tree_closing_notification_recipients = [] - from_address = 'tryserver@chromium.org' - code_review_site = 'https://codereview.chromium.org' - - class Master4a(Base): - """Try server master.""" - master_host = 'master4a.golo.chromium.org' - master_port_base = 14 - tree_closing_notification_recipients = [] - from_address = 'tryserver@chromium.org' - code_review_site = 'https://codereview.chromium.org' - - ## Native Client related - - class NaClBase(Master3): - """Base class for Native Client masters.""" - tree_closing_notification_recipients = ['bradnelson@chromium.org'] - base_app_url = 'https://nativeclient-status.appspot.com' - tree_status_url = base_app_url + '/status' - store_revisions_url = base_app_url + '/revisions' - last_good_url = base_app_url + '/lkgr' - perf_base_url = 'http://build.chromium.org/f/client/perf' - - ## ChromiumOS related - - class ChromiumOSBase(Master2): - """Legacy base class for ChromiumOS masters""" - base_app_url = 'https://chromiumos-status.appspot.com' - tree_status_url = base_app_url + '/status' - store_revisions_url = base_app_url + '/revisions' - last_good_url = base_app_url + '/lkgr' - - class ChromiumOSBase2a(Master2a): - """Base class for ChromiumOS masters""" - base_app_url = 'https://chromiumos-status.appspot.com' - tree_status_url = base_app_url + '/status' - store_revisions_url = base_app_url + '/revisions' - last_good_url = base_app_url + '/lkgr' |