summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorphajdan.jr <phajdan.jr@chromium.org>2015-08-18 04:46:06 -0700
committerCommit bot <commit-bot@chromium.org>2015-08-18 11:46:52 +0000
commit39a1c284b008e3db5cb916ba489a2b65e04c21cc (patch)
tree370ffd1f70b2d22841b98e0f260cac6f61ddca32
parent28fccd5cbdec7e4c1a38d07926591da41331d7d8 (diff)
downloadchromium_src-39a1c284b008e3db5cb916ba489a2b65e04c21cc.zip
chromium_src-39a1c284b008e3db5cb916ba489a2b65e04c21cc.tar.gz
chromium_src-39a1c284b008e3db5cb916ba489a2b65e04c21cc.tar.bz2
runtest.py: remove legacy test-results related code
See https://goto.google.com/vdhuz for context (Google-internal). BUG=506498 Review URL: https://codereview.chromium.org/1302443003 Cr-Commit-Position: refs/heads/master@{#343887}
-rw-r--r--infra/scripts/legacy/scripts/common/chromium_utils.py280
-rwxr-xr-xinfra/scripts/legacy/scripts/common/env.py439
-rw-r--r--infra/scripts/legacy/scripts/common/url_helper.py60
-rw-r--r--infra/scripts/legacy/scripts/slave/bootstrap.py47
-rw-r--r--infra/scripts/legacy/scripts/slave/gtest/__init__.py0
-rw-r--r--infra/scripts/legacy/scripts/slave/gtest/json_results_generator.py255
-rw-r--r--infra/scripts/legacy/scripts/slave/gtest/networktransaction.py46
-rw-r--r--infra/scripts/legacy/scripts/slave/gtest/test_result.py42
-rw-r--r--infra/scripts/legacy/scripts/slave/gtest/test_results_uploader.py32
-rwxr-xr-xinfra/scripts/legacy/scripts/slave/gtest_slave_utils.py103
-rwxr-xr-xinfra/scripts/legacy/scripts/slave/runtest.py114
-rw-r--r--infra/scripts/legacy/site_config/config.py18
-rw-r--r--infra/scripts/legacy/site_config/config_bootstrap.py128
-rw-r--r--infra/scripts/legacy/site_config/config_default.py230
14 files changed, 0 insertions, 1794 deletions
diff --git a/infra/scripts/legacy/scripts/common/chromium_utils.py b/infra/scripts/legacy/scripts/common/chromium_utils.py
index 89ebf8b..9371cb7 100644
--- a/infra/scripts/legacy/scripts/common/chromium_utils.py
+++ b/infra/scripts/legacy/scripts/common/chromium_utils.py
@@ -23,8 +23,6 @@ import threading
import time
import traceback
-from common import env
-
BUILD_DIR = os.path.realpath(os.path.join(
os.path.dirname(__file__), os.pardir, os.pardir))
@@ -472,149 +470,6 @@ def GetCommandOutput(command):
return output
-def ListMasters(cue='master.cfg', include_public=True, include_internal=True):
- """Returns all the masters found."""
- # Look for "internal" masters first.
- path_internal = os.path.join(
- BUILD_DIR, os.pardir, 'build_internal', 'masters/*/' + cue)
- path = os.path.join(BUILD_DIR, 'masters/*/' + cue)
- filenames = []
- if include_public:
- filenames += glob.glob(path)
- if include_internal:
- filenames += glob.glob(path_internal)
- return [os.path.abspath(os.path.dirname(f)) for f in filenames]
-
-
-def MasterPath(mastername, include_public=True, include_internal=True):
- path = os.path.join(BUILD_DIR, 'masters', 'master.%s' % mastername)
- path_internal = os.path.join(
- BUILD_DIR, os.pardir, 'build_internal', 'masters',
- 'master.%s' % mastername)
- if include_public and os.path.isdir(path):
- return path
- if include_internal and os.path.isdir(path_internal):
- return path_internal
- raise LookupError('Path for master %s not found' % mastername)
-
-
-def ListMastersWithSlaves(include_public=True, include_internal=True):
- masters_path = ListMasters('builders.pyl', include_public, include_internal)
- masters_path.extend(ListMasters('slaves.cfg', include_public,
- include_internal))
- return masters_path
-
-
-def GetSlavesFromMasterPath(path, fail_hard=False):
- builders_path = os.path.join(path, 'builders.pyl')
- if os.path.exists(builders_path):
- return GetSlavesFromBuildersFile(builders_path)
- return RunSlavesCfg(os.path.join(path, 'slaves.cfg'), fail_hard=fail_hard)
-
-
-def GetAllSlaves(fail_hard=False, include_public=True, include_internal=True):
- """Return all slave objects from masters."""
- slaves = []
- for master in ListMastersWithSlaves(include_public, include_internal):
- cur_slaves = GetSlavesFromMasterPath(master, fail_hard)
- for slave in cur_slaves:
- slave['mastername'] = os.path.basename(master)
- slaves.extend(cur_slaves)
- return slaves
-
-
-def GetActiveSubdir():
- """Get current checkout's subdir, if checkout uses subdir layout."""
- rootdir, subdir = os.path.split(os.path.dirname(BUILD_DIR))
- if subdir != 'b' and os.path.basename(rootdir) == 'c':
- return subdir
-
-
-def GetActiveSlavename():
- slavename = os.getenv('TESTING_SLAVENAME')
- if not slavename:
- slavename = socket.getfqdn().split('.', 1)[0].lower()
- subdir = GetActiveSubdir()
- if subdir:
- return '%s#%s' % (slavename, subdir)
- return slavename
-
-
-def EntryToSlaveName(entry):
- """Produces slave name from the slaves config dict."""
- name = entry.get('slavename') or entry.get('hostname')
- if 'subdir' in entry:
- return '%s#%s' % (name, entry['subdir'])
- return name
-
-
-def GetActiveMaster(slavename=None, default=None):
- """Returns the name of the Active master serving the current host.
-
- Parse all of the active masters with slaves matching the current hostname
- and optional slavename. Returns |default| if no match found.
- """
- slavename = slavename or GetActiveSlavename()
- for slave in GetAllSlaves():
- if slavename == EntryToSlaveName(slave):
- return slave['master']
- return default
-
-
-@contextmanager
-def MasterEnvironment(master_dir):
- """Context manager that enters an enviornment similar to a master's.
-
- This involves:
- - Modifying 'sys.path' to include paths available to the master.
- - Changing directory (via os.chdir()) to the master's base directory.
-
- These changes will be reverted after the context manager completes.
-
- Args:
- master_dir: (str) The master's base directory.
- """
- master_dir = os.path.abspath(master_dir)
-
- # Setup a 'sys.path' that is adequate for loading 'slaves.cfg'.
- old_cwd = os.getcwd()
-
- with env.GetInfraPythonPath(master_dir=master_dir).Enter():
- try:
- os.chdir(master_dir)
- yield
- finally:
- os.chdir(old_cwd)
-
-
-def ParsePythonCfg(cfg_filepath, fail_hard=False):
- """Retrieves data from a python config file."""
- if not os.path.exists(cfg_filepath):
- return None
-
- # Execute 'slaves.sfg' in the master path environment.
- with MasterEnvironment(os.path.dirname(os.path.abspath(cfg_filepath))):
- try:
- local_vars = {}
- execfile(os.path.join(cfg_filepath), local_vars)
- del local_vars['__builtins__']
- return local_vars
- except Exception as e:
- # pylint: disable=C0323
- print >>sys.stderr, 'An error occurred while parsing %s: %s' % (
- cfg_filepath, e)
- print >>sys.stderr, traceback.format_exc() # pylint: disable=C0323
- if fail_hard:
- raise
- return {}
-
-
-def RunSlavesCfg(slaves_cfg, fail_hard=False):
- """Runs slaves.cfg in a consistent way."""
- slave_config = ParsePythonCfg(slaves_cfg, fail_hard=fail_hard) or {}
- return slave_config.get('slaves', [])
-
-
def convert_json(option, _, value, parser):
"""Provide an OptionParser callback to unmarshal a JSON string."""
setattr(parser.values, option.dest, json.loads(value))
@@ -640,138 +495,3 @@ def AddPropertiesOptions(option_parser):
callback=convert_json, type='string',
nargs=1, default={},
help='factory properties in JSON format')
-
-
-def ReadJsonAsUtf8(filename=None, text=None):
- """Read a json file or string and output a dict.
-
- This function is different from json.load and json.loads in that it
- returns utf8-encoded string for keys and values instead of unicode.
-
- Args:
- filename: path of a file to parse
- text: json string to parse
-
- If both 'filename' and 'text' are provided, 'filename' is used.
- """
- def _decode_list(data):
- rv = []
- for item in data:
- if isinstance(item, unicode):
- item = item.encode('utf-8')
- elif isinstance(item, list):
- item = _decode_list(item)
- elif isinstance(item, dict):
- item = _decode_dict(item)
- rv.append(item)
- return rv
-
- def _decode_dict(data):
- rv = {}
- for key, value in data.iteritems():
- if isinstance(key, unicode):
- key = key.encode('utf-8')
- if isinstance(value, unicode):
- value = value.encode('utf-8')
- elif isinstance(value, list):
- value = _decode_list(value)
- elif isinstance(value, dict):
- value = _decode_dict(value)
- rv[key] = value
- return rv
-
- if filename:
- with open(filename, 'rb') as f:
- return json.load(f, object_hook=_decode_dict)
- if text:
- return json.loads(text, object_hook=_decode_dict)
-
-
-def DatabaseSetup(buildmaster_config, require_dbconfig=False):
- """Read database credentials in the master directory."""
- if os.path.isfile('.dbconfig'):
- values = {}
- execfile('.dbconfig', values)
- if 'password' not in values:
- raise Exception('could not get db password')
-
- buildmaster_config['db_url'] = 'postgresql://%s:%s@%s/%s' % (
- values['username'], values['password'],
- values.get('hostname', 'localhost'), values['dbname'])
- else:
- assert not require_dbconfig
-
-
-def ReadBuildersFile(builders_path):
- with open(builders_path) as fp:
- contents = fp.read()
- return ParseBuildersFileContents(builders_path, contents)
-
-
-def ParseBuildersFileContents(path, contents):
- builders = ast.literal_eval(contents)
-
- # Set some additional derived fields that are derived from the
- # file's location in the filesystem.
- basedir = os.path.dirname(os.path.abspath(path))
- master_dirname = os.path.basename(basedir)
- master_name_comps = master_dirname.split('.')[1:]
- buildbot_path = '.'.join(master_name_comps)
- master_classname = ''.join(c[0].upper() + c[1:] for c in master_name_comps)
- builders['master_dirname'] = master_dirname
- builders.setdefault('master_classname', master_classname)
- builders.setdefault('buildbot_url',
- 'https://build.chromium.org/p/%s/' % buildbot_path)
-
- builders.setdefault('buildbucket_bucket', None)
- builders.setdefault('service_account_file', None)
-
- # The _str fields are printable representations of Python values:
- # if builders['foo'] == "hello", then builders['foo_str'] == "'hello'".
- # This allows them to be read back in by Python scripts properly.
- builders['buildbucket_bucket_str'] = repr(builders['buildbucket_bucket'])
- builders['service_account_file_str'] = repr(builders['service_account_file'])
-
- return builders
-
-
-def GetSlavesFromBuildersFile(builders_path):
- """Read builders_path and return a list of slave dicts."""
- builders = ReadBuildersFile(builders_path)
- return GetSlavesFromBuilders(builders)
-
-
-def GetSlavesFromBuilders(builders):
- """Returns a list of slave dicts derived from the builders dict."""
- builders_in_pool = {}
-
- # builders.pyl contains a list of builders -> slave_pools
- # and a list of slave_pools -> slaves.
- # We require that each slave is in a single pool, but each slave
- # may have multiple builders, so we need to build up the list of
- # builders each slave pool supports.
- for builder_name, builder_vals in builders['builders'].items():
- pool_names = builder_vals['slave_pools']
- for pool_name in pool_names:
- if pool_name not in builders_in_pool:
- builders_in_pool[pool_name] = set()
- pool_data = builders['slave_pools'][pool_name]
- for slave in pool_data['slaves']:
- builders_in_pool[pool_name].add(builder_name)
-
- # Now we can generate the list of slaves using the above lookup table.
- slaves = []
- for pool_name, pool_data in builders['slave_pools'].items():
- slave_data = pool_data['slave_data']
- builder_names = sorted(builders_in_pool[pool_name])
- for slave in pool_data['slaves']:
- slaves.append({
- 'hostname': slave,
- 'builder_name': builder_names,
- 'master': builders['master_classname'],
- 'os': slave_data['os'],
- 'version': slave_data['version'],
- 'bits': slave_data['bits'],
- })
-
- return slaves
diff --git a/infra/scripts/legacy/scripts/common/env.py b/infra/scripts/legacy/scripts/common/env.py
deleted file mode 100755
index 3e497ff..0000000
--- a/infra/scripts/legacy/scripts/common/env.py
+++ /dev/null
@@ -1,439 +0,0 @@
-#!/usr/bin/env python
-# Copyright 2015 The Chromium Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-"""Implements a standard mechanism for Chrome Infra Python environment setup.
-
-This library provides a central location to define Chrome Infra environment
-setup. It also provides several faculties to install this environment.
-
-Within a cooperating script, the environment can be setup by importing this
-module and running its 'Install' method:
-
- # Install Chrome-Infra environment (replaces 'sys.path').
- sys.path.insert(0,
- os.path.join(os.path.dirname(__file__), os.pardir, ...))
- # (/path/to/build/scripts)
- import common.env
- common.env.Install()
-
-When attempting to export the Chrome Infra path to external scripts, this
-script can be invoked as an executable with various subcommands to emit a valid
-PYTHONPATH clause.
-
-In addition, this module has several functions to construct the path.
-
-The goal is to deploy this module universally among Chrome-Infra scripts,
-BuildBot configurations, tool invocations, and tests to ensure that they all
-execute with the same centrally-defined environment.
-"""
-
-import argparse
-import collections
-import contextlib
-import imp
-import itertools
-import os
-import sys
-import traceback
-
-
-# Export for bootstrapping.
-__all__ = [
- 'Install',
- 'PythonPath',
- ]
-
-
-# Name of enviornment extension file to seek.
-ENV_EXTENSION_NAME = 'environment.cfg.py'
-
-# Standard directories (based on this file's location in the <build> tree).
-def path_if(*args):
- if not all(args):
- return None
- path = os.path.abspath(os.path.join(*args))
- return (path) if os.path.exists(path) else (None)
-
-# The path to the <build> directory in which this script resides.
-Build = path_if(os.path.dirname(__file__), os.pardir, os.pardir)
-# The path to the <build_internal> directory.
-BuildInternal = path_if(Build, os.pardir, 'build_internal')
-
-
-def SetPythonPathEnv(value):
- """Sets the system's PYTHONPATH environemnt variable.
-
- Args:
- value (str): The value to use. If this is empty/None, the system's
- PYTHONPATH will be cleared.
- """
- # Since we can't assign None to the environment "dictionary", we have to
- # either set or delete the key depending on the original value.
- if value is not None:
- os.environ['PYTHONPATH'] = str(value)
- else:
- os.environ.pop('PYTHONPATH', None)
-
-
-def Install(**kwargs):
- """Replaces the current 'sys.path' with a hermetic Chrome-Infra path.
-
- Args:
- kwargs (dict): See GetInfraPythonPath arguments.
-
- Returns (PythonPath): The PythonPath object that was installed.
- """
- infra_python_path = GetInfraPythonPath(**kwargs)
- infra_python_path.Install()
- return infra_python_path
-
-
-def SplitPath(path):
- """Returns (list): A list of path elements.
-
- Splits a path into path elements. For example (assuming '/' is the local
- system path separator):
- >>> print SplitPath('/a/b/c/d')
- ['/', 'a', 'b', 'c', 'd']
- >>> print SplitPath('a/b/c')
- ['a', 'b,' 'c']
- """
- parts = []
- while True:
- path, component = os.path.split(path)
- if not component:
- if path:
- parts.append(path)
- break
- parts.append(component)
- parts.reverse()
- return parts
-
-
-def ExtendPath(base, root_dir):
- """Returns (PythonPath): The extended python path.
-
- This method looks for the ENV_EXTENSION_NAME file within "root_dir". If
- present, it will be loaded as a Python module and have its "Extend" method
- called.
-
- If no extension is found, the base PythonPath will be returned.
-
- Args:
- base (PythonPath): The base python path.
- root_dir (str): The path to check for an extension.
- """
- extension_path = os.path.join(root_dir, ENV_EXTENSION_NAME)
- if not os.path.isfile(extension_path):
- return base
- with open(extension_path, 'r') as fd:
- extension = fd.read()
- extension_module = imp.new_module('env-extension')
-
- # Execute the enviornment extension.
- try:
- exec extension in extension_module.__dict__
-
- extend_func = getattr(extension_module, 'Extend', None)
- assert extend_func, (
- "The environment extension module is missing the 'Extend()' method.")
- base = extend_func(base, root_dir)
- if not isinstance(base, PythonPath):
- raise TypeError("Extension module returned non-PythonPath object (%s)" % (
- type(base).__name__,))
- except Exception:
- # Re-raise the exception, but include the configuration file name.
- tb = traceback.format_exc()
- raise RuntimeError("Environment extension [%s] raised exception: %s" % (
- extension_path, tb))
- return base
-
-
-def IsSystemPythonPath(path):
- """Returns (bool): If a python path is user-installed.
-
- Paths that are known to be user-installed paths can be ignored when setting
- up a hermetic Python path environment to avoid user libraries that would not
- be present in other environments falsely affecting code.
-
- This function can be updated as-needed to exclude other non-system paths
- encountered on bots and in the wild.
- """
- components = SplitPath(path)
- for component in components:
- if component in ('dist-packages', 'site-packages'):
- return False
- return True
-
-
-class PythonPath(collections.Sequence):
- """An immutable set of Python path elements.
-
- All paths represented in this structure are absolute. If a relative path
- is passed into this structure, it will be converted to absolute based on
- the current working directory (via os.path.abspath).
- """
-
- def __init__(self, components=None):
- """Initializes a new PythonPath instance.
-
- Args:
- components (list): A list of path component strings.
- """
- seen = set()
- self._components = []
- for component in (components or ()):
- component = os.path.abspath(component)
- assert isinstance(component, basestring), (
- "Path component '%s' is not a string (%s)" % (
- component, type(component).__name__))
- if component in seen:
- continue
- seen.add(component)
- self._components.append(component)
-
- def __getitem__(self, value):
- return self._components[value]
-
- def __len__(self):
- return len(self._components)
-
- def __iadd__(self, other):
- return self.Append(other)
-
- def __repr__(self):
- return self.pathstr
-
- def __eq__(self, other):
- assert isinstance(other, type(self))
- return self._components == other._components
-
- @classmethod
- def Flatten(cls, *paths):
- """Returns (list): A single-level list containing flattened path elements.
-
- >>> print PythonPath.Flatten('a', ['b', ['c', 'd']])
- ['a', 'b', 'c', 'd']
- """
- result = []
- for path in paths:
- if not isinstance(path, basestring):
- # Assume it's an iterable of paths.
- result += cls.Flatten(*path)
- else:
- result.append(path)
- return result
-
- @classmethod
- def FromPaths(cls, *paths):
- """Returns (PythonPath): A PythonPath instantiated from path elements.
-
- Args:
- paths (tuple): A tuple of path elements or iterables containing path
- elements (e.g., PythonPath instances).
- """
- return cls(cls.Flatten(*paths))
-
- @classmethod
- def FromPathStr(cls, pathstr):
- """Returns (PythonPath): A PythonPath instantiated from the path string.
-
- Args:
- pathstr (str): An os.pathsep()-delimited path string.
- """
- return cls(pathstr.split(os.pathsep))
-
- @property
- def pathstr(self):
- """Returns (str): A path string for the instance's path elements."""
- return os.pathsep.join(self)
-
- def IsHermetic(self):
- """Returns (bool): True if this instance contains only system paths."""
- return all(IsSystemPythonPath(p) for p in self)
-
- def GetHermetic(self):
- """Returns (PythonPath): derivative PythonPath containing only system paths.
- """
- return type(self).FromPaths(*(p for p in self if IsSystemPythonPath(p)))
-
- def Append(self, *paths):
- """Returns (PythonPath): derivative PythonPath with paths added to the end.
-
- Args:
- paths (tuple): A tuple of path elements to append to the current instance.
- """
- return type(self)(itertools.chain(self, self.FromPaths(*paths)))
-
- def Override(self, *paths):
- """Returns (PythonPath): derivative PythonPath with paths prepended.
-
- Args:
- paths (tuple): A tuple of path elements to prepend to the current
- instance.
- """
- return self.FromPaths(*paths).Append(self)
-
- def Install(self):
- """Overwrites Python runtime variables based on the current instance.
-
- Performs the following operations:
- - Replaces sys.path with the current instance's path.
- - Replaces os.environ['PYTHONPATH'] with the current instance's path
- string.
- """
- sys.path = list(self)
- SetPythonPathEnv(self.pathstr)
-
- @contextlib.contextmanager
- def Enter(self):
- """Context manager wrapper for Install.
-
- On exit, the context manager will restore the original environment.
- """
- orig_sys_path = sys.path[:]
- orig_pythonpath = os.environ.get('PYTHONPATH')
-
- try:
- self.Install()
- yield
- finally:
- sys.path = orig_sys_path
- SetPythonPathEnv(orig_pythonpath)
-
-
-def GetSysPythonPath(hermetic=True):
- """Returns (PythonPath): A path based on 'sys.path'.
-
- Args:
- hermetic (bool): If True, prune any non-system path.
- """
- path = PythonPath.FromPaths(*sys.path)
- if hermetic:
- path = path.GetHermetic()
- return path
-
-
-def GetEnvPythonPath():
- """Returns (PythonPath): A path based on the PYTHONPATH environment variable.
- """
- pythonpath = os.environ.get('PYTHONPATH')
- if not pythonpath:
- return PythonPath.FromPaths()
- return PythonPath.FromPathStr(pythonpath)
-
-
-def GetMasterPythonPath(master_dir):
- """Returns (PythonPath): A path including a BuildBot master's directory.
-
- Args:
- master_dir (str): The BuildBot master root directory.
- """
- return PythonPath.FromPaths(master_dir)
-
-
-def GetBuildPythonPath():
- """Returns (PythonPath): The Chrome Infra build path."""
- build_path = PythonPath.FromPaths()
- for extension_dir in (
- Build,
- BuildInternal,
- ):
- if extension_dir:
- build_path = ExtendPath(build_path, extension_dir)
- return build_path
-
-
-def GetInfraPythonPath(hermetic=True, master_dir=None):
- """Returns (PythonPath): The full working Chrome Infra utility path.
-
- This path is consistent for master, slave, and tool usage. It includes (in
- this order):
- - Any environment PYTHONPATH overrides.
- - If 'master_dir' is supplied, the master's python path component.
- - The Chrome Infra build path.
- - The system python path.
-
- Args:
- hermetic (bool): True, prune any non-system path from the system path.
- master_dir (str): If not None, include a master path component.
- """
- path = GetEnvPythonPath()
- if master_dir:
- path += GetMasterPythonPath(master_dir)
- path += GetBuildPythonPath()
- path += GetSysPythonPath(hermetic=hermetic)
- return path
-
-
-def _InfraPathFromArgs(args):
- """Returns (PythonPath): A PythonPath populated from command-line arguments.
-
- Args:
- args (argparse.Namespace): The command-line arguments constructed by 'main'.
- """
- return GetInfraPythonPath(
- master_dir=args.master_dir,
- )
-
-
-def _Command_Echo(args, path):
- """Returns (int): Return code.
-
- Command function for the 'echo' subcommand. Outputs the path string for
- 'path'.
-
- Args:
- args (argparse.Namespace): The command-line arguments constructed by 'main'.
- path (PythonPath): The python path to use.
- """
- args.output.write(path.pathstr)
- return 0
-
-
-def _Command_Print(args, path):
- """Returns (int): Return code.
-
- Command function for the 'print' subcommand. Outputs each path component in
- path on a separate line.
-
- Args:
- args (argparse.Namespace): The command-line arguments constructed by 'main'.
- path (PythonPath): The python path to use.
- """
- for component in path:
- print >>args.output, component
- return 0
-
-
-def main():
- """Main execution function."""
- parser = argparse.ArgumentParser()
- parser.add_argument('-M', '--master_dir',
- help="Augment the path with the master's directory.")
- parser.add_argument('-o', '--output', metavar='PATH',
- type=argparse.FileType('w'), default='-',
- help="File to output to (use '-' for STDOUT).")
-
- subparsers = parser.add_subparsers()
-
- # 'echo'
- subparser = subparsers.add_parser('echo')
- subparser.set_defaults(func=_Command_Echo)
-
- # 'print'
- subparser = subparsers.add_parser('print')
- subparser.set_defaults(func=_Command_Print)
-
- # Parse
- args = parser.parse_args()
-
- # Execute our subcommand function, which will return the exit code.
- path = _InfraPathFromArgs(args)
- return args.func(args, path)
-
-
-if __name__ == '__main__':
- sys.exit(main())
diff --git a/infra/scripts/legacy/scripts/common/url_helper.py b/infra/scripts/legacy/scripts/common/url_helper.py
deleted file mode 100644
index 78adc86..0000000
--- a/infra/scripts/legacy/scripts/common/url_helper.py
+++ /dev/null
@@ -1,60 +0,0 @@
-# Copyright 2013 The Chromium Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-import mimetypes
-import urllib2
-
-
-def GetMimeType(filename):
- return mimetypes.guess_type(filename)[0] or 'application/octet-stream'
-
-
-def EncodeMultipartFormData(fields, files):
- """Encode form fields for multipart/form-data.
-
- Args:
- fields: A sequence of (name, value) elements for regular form fields.
- files: A sequence of (name, filename, value) elements for data to be
- uploaded as files.
- Returns:
- (content_type, body) ready for httplib.HTTP instance.
-
- Source:
- http://code.google.com/p/rietveld/source/browse/trunk/upload.py
- """
- BOUNDARY = '-M-A-G-I-C---B-O-U-N-D-A-R-Y-'
- CRLF = '\r\n'
- lines = []
-
- for key, value in fields:
- lines.append('--' + BOUNDARY)
- lines.append('Content-Disposition: form-data; name="%s"' % key)
- lines.append('')
- if isinstance(value, unicode):
- value = value.encode('utf-8')
- lines.append(value)
-
- for key, filename, value in files:
- lines.append('--' + BOUNDARY)
- lines.append('Content-Disposition: form-data; name="%s"; filename="%s"' %
- (key, filename))
- lines.append('Content-Type: %s' % GetMimeType(filename))
- lines.append('')
- if isinstance(value, unicode):
- value = value.encode('utf-8')
- lines.append(value)
-
- lines.append('--' + BOUNDARY + '--')
- lines.append('')
- body = CRLF.join(lines)
- content_type = 'multipart/form-data; boundary=%s' % BOUNDARY
- return content_type, body
-
-
-def upload_files(url, attrs, file_objs):
- content_type, data = EncodeMultipartFormData(attrs, file_objs)
- headers = {"Content-Type": content_type}
- request = urllib2.Request(url, data, headers)
-
- return urllib2.urlopen(request)
diff --git a/infra/scripts/legacy/scripts/slave/bootstrap.py b/infra/scripts/legacy/scripts/slave/bootstrap.py
deleted file mode 100644
index 2bfa4ce..0000000
--- a/infra/scripts/legacy/scripts/slave/bootstrap.py
+++ /dev/null
@@ -1,47 +0,0 @@
-# Copyright (c) 2012 The Chromium Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-"""Utilities to enable slaves to determine their master without importing any
-buildbot or twisted code.
-"""
-
-import inspect
-import os
-import sys
-
-from common import chromium_utils
-import config_bootstrap
-
-
-def ImportMasterConfigs(master_name=None, include_internal=True):
- """Imports master configs.
-
- Normally a slave can use chromium_utils.GetActiveMaster() to find
- itself and determine which ActiveMaster to use. In that case, the
- active master name is passed in as an arg, and we only load the
- site_config.py that defines it. When testing, the current "slave"
- won't be found. In that case, we don't know which config to use, so
- load them all. In either case, masters are assigned as attributes
- to the config.Master object.
- """
- for master in chromium_utils.ListMasters(include_internal=include_internal):
- path = os.path.join(master, 'master_site_config.py')
- if os.path.exists(path):
- local_vars = {}
- try:
- execfile(path, local_vars)
- # pylint: disable=W0703
- except Exception, e:
- # Naked exceptions are banned by the style guide but we are
- # trying to be resilient here.
- print >> sys.stderr, 'WARNING: cannot exec ' + path
- print >> sys.stderr, e
- for (symbol_name, symbol) in local_vars.iteritems():
- if inspect.isclass(symbol):
- setattr(symbol, 'local_config_path', master)
- setattr(config_bootstrap.Master, symbol_name, symbol)
- # If we have a master_name and it matches, set
- # config_bootstrap.Master.active_master.
- if master_name and master_name == symbol_name:
- setattr(config_bootstrap.Master, 'active_master', symbol)
diff --git a/infra/scripts/legacy/scripts/slave/gtest/__init__.py b/infra/scripts/legacy/scripts/slave/gtest/__init__.py
deleted file mode 100644
index e69de29..0000000
--- a/infra/scripts/legacy/scripts/slave/gtest/__init__.py
+++ /dev/null
diff --git a/infra/scripts/legacy/scripts/slave/gtest/json_results_generator.py b/infra/scripts/legacy/scripts/slave/gtest/json_results_generator.py
deleted file mode 100644
index cf207b0..0000000
--- a/infra/scripts/legacy/scripts/slave/gtest/json_results_generator.py
+++ /dev/null
@@ -1,255 +0,0 @@
-# Copyright (c) 2012 The Chromium Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-"""A utility class to generate JSON results from given test results and upload
-them to the specified results server.
-
-"""
-
-from __future__ import with_statement
-
-import codecs
-import logging
-import os
-import time
-
-import simplejson
-from slave.gtest.test_result import TestResult
-from slave.gtest.test_results_uploader import TestResultsUploader
-
-# A JSON results generator for generic tests.
-
-JSON_PREFIX = 'ADD_RESULTS('
-JSON_SUFFIX = ');'
-
-
-def test_did_pass(test_result):
- return not test_result.failed and test_result.modifier == TestResult.NONE
-
-
-def add_path_to_trie(path, value, trie):
- """Inserts a single flat directory path and associated value into a directory
- trie structure."""
- if not '/' in path:
- trie[path] = value
- return
-
- # we don't use slash
- # pylint: disable=W0612
- directory, slash, rest = path.partition('/')
- if not directory in trie:
- trie[directory] = {}
- add_path_to_trie(rest, value, trie[directory])
-
-
-def generate_test_timings_trie(individual_test_timings):
- """Breaks a test name into chunks by directory and puts the test time as a
- value in the lowest part, e.g.
- foo/bar/baz.html: 1ms
- foo/bar/baz1.html: 3ms
-
- becomes
- foo: {
- bar: {
- baz.html: 1,
- baz1.html: 3
- }
- }
- """
- trie = {}
- # Only use the timing of the first try of each test.
- for test_results in individual_test_timings:
- test = test_results[0].test_name
-
- add_path_to_trie(test, int(1000 * test_results[-1].test_run_time), trie)
-
- return trie
-
-
-class JSONResultsGenerator(object):
- """A JSON results generator for generic tests."""
-
- FAIL_LABEL = 'FAIL'
- PASS_LABEL = 'PASS'
- FLAKY_LABEL = ' '.join([FAIL_LABEL, PASS_LABEL])
- SKIP_LABEL = 'SKIP'
-
- ACTUAL = 'actual'
- BLINK_REVISION = 'blink_revision'
- BUILD_NUMBER = 'build_number'
- BUILDER_NAME = 'builder_name'
- CHROMIUM_REVISION = 'chromium_revision'
- EXPECTED = 'expected'
- FAILURE_SUMMARY = 'num_failures_by_type'
- SECONDS_SINCE_EPOCH = 'seconds_since_epoch'
- TEST_TIME = 'time'
- TESTS = 'tests'
- VERSION = 'version'
- VERSION_NUMBER = 3
-
- RESULTS_FILENAME = 'results.json'
- TIMES_MS_FILENAME = 'times_ms.json'
- FULL_RESULTS_FILENAME = 'full_results.json'
-
- def __init__(self, builder_name, build_name, build_number,
- results_file_base_path, builder_base_url,
- test_results_map, svn_revisions=None,
- test_results_server=None,
- test_type='',
- master_name='',
- file_writer=None):
- """Modifies the results.json file. Grabs it off the archive directory
- if it is not found locally.
-
- Args
- builder_name: the builder name (e.g. Webkit).
- build_name: the build name (e.g. webkit-rel).
- build_number: the build number.
- results_file_base_path: Absolute path to the directory containing the
- results json file.
- builder_base_url: the URL where we have the archived test results.
- If this is None no archived results will be retrieved.
- test_results_map: A dictionary that maps test_name to a list of
- TestResult, one for each time the test was retried.
- svn_revisions: A (json_field_name, revision) pair for SVN
- repositories that tests rely on. The SVN revision will be
- included in the JSON with the given json_field_name.
- test_results_server: server that hosts test results json.
- test_type: test type string (e.g. 'layout-tests').
- master_name: the name of the buildbot master.
- file_writer: if given the parameter is used to write JSON data to a file.
- The parameter must be the function that takes two arguments, 'file_path'
- and 'data' to be written into the file_path.
- """
- self._builder_name = builder_name
- self._build_name = build_name
- self._build_number = build_number
- self._builder_base_url = builder_base_url
- self._results_directory = results_file_base_path
-
- self._test_results_map = test_results_map
-
- self._svn_revisions = svn_revisions
- if not self._svn_revisions:
- self._svn_revisions = {}
-
- self._test_results_server = test_results_server
- self._test_type = test_type
- self._master_name = master_name
- self._file_writer = file_writer
-
- def generate_json_output(self):
- json = self.get_full_results_json()
- if json:
- file_path = os.path.join(self._results_directory,
- self.FULL_RESULTS_FILENAME)
- self._write_json(json, file_path)
-
- def generate_times_ms_file(self):
- times = generate_test_timings_trie(self._test_results_map.values())
- file_path = os.path.join(self._results_directory, self.TIMES_MS_FILENAME)
- self._write_json(times, file_path)
-
- def get_full_results_json(self):
- results = {self.VERSION: self.VERSION_NUMBER}
-
- # Metadata generic to all results.
- results[self.BUILDER_NAME] = self._builder_name
- results[self.BUILD_NUMBER] = self._build_number
- results[self.SECONDS_SINCE_EPOCH] = int(time.time())
- for name, revision in self._svn_revisions:
- results[name + '_revision'] = revision
-
- tests = results.setdefault(self.TESTS, {})
- for test_name in self._test_results_map.iterkeys():
- tests[test_name] = self._make_test_data(test_name)
-
- self._insert_failure_map(results)
-
- return results
-
- def _insert_failure_map(self, results):
- # FAIL, PASS, NOTRUN
- summary = {self.PASS_LABEL: 0, self.FAIL_LABEL: 0, self.SKIP_LABEL: 0}
- for test_results in self._test_results_map.itervalues():
- # Use the result of the first test for aggregate statistics. This may
- # count as failing a test that passed on retry, but it's a more useful
- # statistic and it's consistent with our other test harnesses.
- test_result = test_results[0]
- if test_did_pass(test_result):
- summary[self.PASS_LABEL] += 1
- elif test_result.modifier == TestResult.DISABLED:
- summary[self.SKIP_LABEL] += 1
- elif test_result.failed:
- summary[self.FAIL_LABEL] += 1
-
- results[self.FAILURE_SUMMARY] = summary
-
- def _make_test_data(self, test_name):
- test_data = {}
- expected, actual = self._get_expected_and_actual_results(test_name)
- test_data[self.EXPECTED] = expected
- test_data[self.ACTUAL] = actual
- # Use the timing of the first try, it's a better representative since it
- # runs under more load than retries.
- run_time = int(self._test_results_map[test_name][0].test_run_time)
- test_data[self.TEST_TIME] = run_time
-
- return test_data
-
- def _get_expected_and_actual_results(self, test_name):
- test_results = self._test_results_map[test_name]
- # Use the modifier of the first try, they should all be the same.
- modifier = test_results[0].modifier
-
- if modifier == TestResult.DISABLED:
- return (self.SKIP_LABEL, self.SKIP_LABEL)
-
- actual_list = []
- for test_result in test_results:
- label = self.FAIL_LABEL if test_result.failed else self.PASS_LABEL
- actual_list.append(label)
- actual = " ".join(actual_list)
-
- if modifier == TestResult.NONE:
- return (self.PASS_LABEL, actual)
- if modifier == TestResult.FLAKY:
- return (self.FLAKY_LABEL, actual)
- if modifier == TestResult.FAILS:
- return (self.FAIL_LABEL, actual)
-
- def upload_json_files(self, json_files):
- """Uploads the given json_files to the test_results_server (if the
- test_results_server is given)."""
- if not self._test_results_server:
- return
-
- if not self._master_name:
- logging.error('--test-results-server was set, but --master-name was not. '
- 'Not uploading JSON files.')
- return
-
- print 'Uploading JSON files for builder: %s' % self._builder_name
- attrs = [('builder', self._builder_name),
- ('testtype', self._test_type),
- ('master', self._master_name)]
-
- files = [(f, os.path.join(self._results_directory, f)) for f in json_files]
-
- uploader = TestResultsUploader(self._test_results_server)
- # Set uploading timeout in case appengine server is having problem.
- # 120 seconds are more than enough to upload test results.
- uploader.upload(attrs, files, 120)
-
- print 'JSON files uploaded.'
-
- def _write_json(self, json_object, file_path):
- # Specify separators in order to get compact encoding.
- json_data = simplejson.dumps(json_object, separators=(',', ':'))
- json_string = json_data
- if self._file_writer:
- self._file_writer(file_path, json_string)
- else:
- with codecs.open(file_path, 'w', 'utf8') as f:
- f.write(json_string)
diff --git a/infra/scripts/legacy/scripts/slave/gtest/networktransaction.py b/infra/scripts/legacy/scripts/slave/gtest/networktransaction.py
deleted file mode 100644
index 10327cb..0000000
--- a/infra/scripts/legacy/scripts/slave/gtest/networktransaction.py
+++ /dev/null
@@ -1,46 +0,0 @@
-# Copyright (c) 2011 The Chromium Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-import logging
-import time
-import urllib2
-
-
-class NetworkTimeout(Exception):
- pass
-
-
-class NetworkTransaction(object):
- def __init__(self, initial_backoff_seconds=10, grown_factor=1.5,
- timeout_seconds=(10 * 60), convert_404_to_None=False):
- self._initial_backoff_seconds = initial_backoff_seconds
- self._backoff_seconds = initial_backoff_seconds
- self._grown_factor = grown_factor
- self._timeout_seconds = timeout_seconds
- self._convert_404_to_None = convert_404_to_None
- self._total_sleep = 0
-
- def run(self, request):
- self._total_sleep = 0
- self._backoff_seconds = self._initial_backoff_seconds
- while True:
- try:
- return request()
- except urllib2.HTTPError, e:
- if self._convert_404_to_None and e.code == 404:
- return None
- self._check_for_timeout()
- logging.warn("Received HTTP status %s loading \"%s\". "
- "Retrying in %s seconds...",
- e.code, e.filename, self._backoff_seconds)
- self._sleep()
-
- def _check_for_timeout(self):
- if self._total_sleep + self._backoff_seconds > self._timeout_seconds:
- raise NetworkTimeout()
-
- def _sleep(self):
- time.sleep(self._backoff_seconds)
- self._total_sleep += self._backoff_seconds
- self._backoff_seconds *= self._grown_factor
diff --git a/infra/scripts/legacy/scripts/slave/gtest/test_result.py b/infra/scripts/legacy/scripts/slave/gtest/test_result.py
deleted file mode 100644
index 3efd37d..0000000
--- a/infra/scripts/legacy/scripts/slave/gtest/test_result.py
+++ /dev/null
@@ -1,42 +0,0 @@
-# Copyright (c) 2012 The Chromium Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-
-def canonical_name(name):
- new_name = name.replace('FLAKY_', '', 1)
- new_name = new_name.replace('FAILS_', '', 1)
- new_name = new_name.replace('DISABLED_', '', 1)
- return new_name
-
-
-class TestResult(object):
- """A simple class that represents a single test result."""
-
- # Test modifier constants.
- (NONE, FAILS, FLAKY, DISABLED) = range(4)
-
- def __init__(self, test, failed, not_run=False, elapsed_time=0):
- self.test_name = canonical_name(test)
- self.failed = failed
- self.test_run_time = elapsed_time
-
- test_name = test
- try:
- test_name = test.split('.')[1]
- except IndexError:
- pass
-
- if not_run:
- self.modifier = self.DISABLED
- elif test_name.startswith('FAILS_'):
- self.modifier = self.FAILS
- elif test_name.startswith('FLAKY_'):
- self.modifier = self.FLAKY
- elif test_name.startswith('DISABLED_'):
- self.modifier = self.DISABLED
- else:
- self.modifier = self.NONE
-
- def fixable(self):
- return self.failed or self.modifier == self.DISABLED
diff --git a/infra/scripts/legacy/scripts/slave/gtest/test_results_uploader.py b/infra/scripts/legacy/scripts/slave/gtest/test_results_uploader.py
deleted file mode 100644
index ead70fb..0000000
--- a/infra/scripts/legacy/scripts/slave/gtest/test_results_uploader.py
+++ /dev/null
@@ -1,32 +0,0 @@
-# Copyright (c) 2011 The Chromium Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-import codecs
-import socket
-
-from common import url_helper
-from slave.gtest.networktransaction import NetworkTransaction
-
-
-class TestResultsUploader(object):
- def __init__(self, host):
- self._host = host
-
- def _upload_files(self, attrs, file_objs):
- url = "http://%s/testfile/upload" % self._host
- url_helper.upload_files(url, attrs, file_objs)
-
- def upload(self, params, files, timeout_seconds):
- file_objs = []
- for filename, path in files:
- with codecs.open(path, "rb") as f:
- file_objs.append(('file', filename, f.read()))
-
- orig_timeout = socket.getdefaulttimeout()
- try:
- socket.setdefaulttimeout(timeout_seconds)
- NetworkTransaction(timeout_seconds=timeout_seconds).run(
- lambda: self._upload_files(params, file_objs))
- finally:
- socket.setdefaulttimeout(orig_timeout)
diff --git a/infra/scripts/legacy/scripts/slave/gtest_slave_utils.py b/infra/scripts/legacy/scripts/slave/gtest_slave_utils.py
deleted file mode 100755
index 2b4135c..0000000
--- a/infra/scripts/legacy/scripts/slave/gtest_slave_utils.py
+++ /dev/null
@@ -1,103 +0,0 @@
-#!/usr/bin/env python
-# Copyright (c) 2012 The Chromium Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-import logging
-import os
-
-from common import gtest_utils
-from slave.gtest.json_results_generator import JSONResultsGenerator
-from slave.gtest.test_result import canonical_name
-from slave.gtest.test_result import TestResult
-
-
-GENERATE_JSON_RESULTS_OPTIONS = [
- 'builder_name', 'build_name', 'build_number', 'results_directory',
- 'builder_base_url', 'webkit_revision', 'chrome_revision',
- 'test_results_server', 'test_type', 'master_name']
-
-FULL_RESULTS_FILENAME = 'full_results.json'
-TIMES_MS_FILENAME = 'times_ms.json'
-
-
-def GetResultsMap(observer):
- """Returns a map of TestResults."""
-
- test_results_map = dict()
- tests = (observer.FailedTests(include_fails=True, include_flaky=True) +
- observer.PassedTests())
- for test in tests:
- key = canonical_name(test)
- test_results_map[key] = []
- tries = observer.TriesForTest(test)
- for test_try in tries:
- # FIXME: Store the actual failure type so we can expose whether the test
- # crashed or timed out. See crbug.com/249965.
- failed = (test_try != gtest_utils.TEST_SUCCESS_LABEL)
- test_results_map[key].append(TestResult(test, failed=failed))
-
- return test_results_map
-
-
-def GenerateJSONResults(test_results_map, options):
- """Generates a JSON results file from the given test_results_map,
- returning the associated generator for use with UploadJSONResults, below.
-
- Args:
- test_results_map: A map of TestResult.
- options: options for json generation. See GENERATE_JSON_RESULTS_OPTIONS
- and OptionParser's help messages below for expected options and their
- details.
- """
-
- if not test_results_map:
- logging.warn('No input results map was given.')
- return
-
- # Make sure we have all the required options (set empty string otherwise).
- for opt in GENERATE_JSON_RESULTS_OPTIONS:
- if not getattr(options, opt, None):
- logging.warn('No value is given for option %s', opt)
- setattr(options, opt, '')
-
- try:
- int(options.build_number)
- except ValueError:
- logging.error('options.build_number needs to be a number: %s',
- options.build_number)
- return
-
- if not os.path.exists(options.results_directory):
- os.makedirs(options.results_directory)
-
- print('Generating json: '
- 'builder_name:%s, build_name:%s, build_number:%s, '
- 'results_directory:%s, builder_base_url:%s, '
- 'webkit_revision:%s, chrome_revision:%s '
- 'test_results_server:%s, test_type:%s, master_name:%s' %
- (options.builder_name, options.build_name, options.build_number,
- options.results_directory, options.builder_base_url,
- options.webkit_revision, options.chrome_revision,
- options.test_results_server, options.test_type,
- options.master_name))
-
- generator = JSONResultsGenerator(
- options.builder_name, options.build_name, options.build_number,
- options.results_directory, options.builder_base_url,
- test_results_map,
- svn_revisions=(('blink', options.webkit_revision),
- ('chromium', options.chrome_revision)),
- test_results_server=options.test_results_server,
- test_type=options.test_type,
- master_name=options.master_name)
- generator.generate_json_output()
- generator.generate_times_ms_file()
- return generator
-
-def UploadJSONResults(generator):
- """Conditionally uploads the results from GenerateJSONResults if
- test_results_server was given."""
- if generator:
- generator.upload_json_files([FULL_RESULTS_FILENAME,
- TIMES_MS_FILENAME])
diff --git a/infra/scripts/legacy/scripts/slave/runtest.py b/infra/scripts/legacy/scripts/slave/runtest.py
index 42ee7db..3e09630 100755
--- a/infra/scripts/legacy/scripts/slave/runtest.py
+++ b/infra/scripts/legacy/scripts/slave/runtest.py
@@ -23,12 +23,8 @@ import sys
from common import chromium_utils
from common import gtest_utils
-# TODO(crbug.com/403564). We almost certainly shouldn't be importing this.
-import config
-
from slave import annotation_utils
from slave import build_directory
-from slave import gtest_slave_utils
from slave import slave_utils
from slave import xvfb
@@ -36,9 +32,6 @@ USAGE = '%s [options] test.exe [test args]' % os.path.basename(sys.argv[0])
CHROME_SANDBOX_PATH = '/opt/chromium/chrome_sandbox'
-# Directory to write JSON for test results into.
-DEST_DIR = 'gtest_results'
-
# The directory that this script is in.
BASE_DIR = os.path.dirname(os.path.abspath(__file__))
@@ -131,95 +124,6 @@ def _RunGTestCommand(
return chromium_utils.RunCommand(command, pipes=pipes, env=env)
-def _GetMaster():
- """Return the master name for the current host."""
- return chromium_utils.GetActiveMaster()
-
-
-def _GetMasterString(master):
- """Returns a message describing what the master is."""
- return '[Running for master: "%s"]' % master
-
-
-def _GenerateJSONForTestResults(options, log_processor):
- """Generates or updates a JSON file from the gtest results XML and upload the
- file to the archive server.
-
- The archived JSON file will be placed at:
- www-dir/DEST_DIR/buildname/testname/results.json
- on the archive server. NOTE: This will be deprecated.
-
- Args:
- options: command-line options that are supposed to have build_dir,
- results_directory, builder_name, build_name and test_output_xml values.
- log_processor: An instance of PerformanceLogProcessor or similar class.
-
- Returns:
- True upon success, False upon failure.
- """
- results_map = None
- try:
- results_map = gtest_slave_utils.GetResultsMap(log_processor)
- except Exception as e:
- # This error will be caught by the following 'not results_map' statement.
- print 'Error: ', e
-
- if not results_map:
- print 'No data was available to update the JSON results'
- # Consider this non-fatal.
- return True
-
- build_dir = os.path.abspath(options.build_dir)
- slave_name = options.builder_name or slave_utils.SlaveBuildName(build_dir)
-
- generate_json_options = copy.copy(options)
- generate_json_options.build_name = slave_name
- generate_json_options.input_results_xml = options.test_output_xml
- generate_json_options.builder_base_url = '%s/%s/%s/%s' % (
- config.Master.archive_url, DEST_DIR, slave_name, options.test_type)
- generate_json_options.master_name = options.master_class_name or _GetMaster()
- generate_json_options.test_results_server = config.Master.test_results_server
-
- print _GetMasterString(generate_json_options.master_name)
-
- generator = None
-
- try:
- if options.revision:
- generate_json_options.chrome_revision = options.revision
- else:
- generate_json_options.chrome_revision = ''
-
- if options.webkit_revision:
- generate_json_options.webkit_revision = options.webkit_revision
- else:
- generate_json_options.webkit_revision = ''
-
- # Generate results JSON file and upload it to the appspot server.
- generator = gtest_slave_utils.GenerateJSONResults(
- results_map, generate_json_options)
-
- except Exception as e:
- print 'Unexpected error while generating JSON: %s' % e
- sys.excepthook(*sys.exc_info())
- return False
-
- # The code can throw all sorts of exceptions, including
- # slave.gtest.networktransaction.NetworkTimeout so just trap everything.
- # Earlier versions of this code ignored network errors, so until a
- # retry mechanism is added, continue to do so rather than reporting
- # an error.
- try:
- # Upload results JSON file to the appspot server.
- gtest_slave_utils.UploadJSONResults(generator)
- except Exception as e:
- # Consider this non-fatal for the moment.
- print 'Unexpected error while uploading JSON: %s' % e
- sys.excepthook(*sys.exc_info())
-
- return True
-
-
def _BuildTestBinaryCommand(_build_dir, test_exe_path, options):
"""Builds a command to run a test binary.
@@ -381,11 +285,6 @@ def _Main(options, args, extra_env):
log_processor = gtest_utils.GTestJSONParser(
options.build_properties.get('mastername'))
- if options.generate_json_file:
- if os.path.exists(options.test_output_xml):
- # remove the old XML output file.
- os.remove(options.test_output_xml)
-
try:
# TODO(dpranke): checking on test_exe is a temporary hack until we
# can change the buildbot master to pass --xvfb instead of --no-xvfb
@@ -426,10 +325,6 @@ def _Main(options, args, extra_env):
_SymbolizeSnippetsInJSON(options, json_file_name)
log_processor.ProcessJSONFile(options.build_dir)
- if options.generate_json_file:
- if not _GenerateJSONForTestResults(options, log_processor):
- return 1
-
if options.annotate:
annotation_utils.annotate(options.test_type, result, log_processor)
@@ -708,15 +603,6 @@ def main():
extra_env['GTEST_TOTAL_SHARDS'] = str(options.total_shards)
extra_env['GTEST_SHARD_INDEX'] = str(options.shard_index - 1)
- if options.results_directory:
- options.test_output_xml = os.path.normpath(os.path.abspath(os.path.join(
- options.results_directory, '%s.xml' % options.test_type)))
- args.append('--gtest_output=xml:' + options.test_output_xml)
- elif options.generate_json_file:
- option_parser.error(
- '--results-directory is required with --generate-json-file=True')
- return 1
-
return _Main(options, args, extra_env)
finally:
if did_launch_dbus:
diff --git a/infra/scripts/legacy/site_config/config.py b/infra/scripts/legacy/site_config/config.py
deleted file mode 100644
index 0108e65..0000000
--- a/infra/scripts/legacy/site_config/config.py
+++ /dev/null
@@ -1,18 +0,0 @@
-# Copyright (c) 2012 The Chromium Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-"""Declares a number of site-dependent variables for use by scripts.
-"""
-
-import os
-
-# DatabaseSetup was moved. Import it for backward compatibility
-from common.chromium_utils import DatabaseSetup # pylint: disable=W0611
-from config_bootstrap import config_private # pylint: disable=W0403,W0611
-from config_bootstrap import Master # pylint: disable=W0403,W0611
-
-SITE_CONFIG_PATH = os.path.abspath(os.path.dirname(__file__))
-
-def SiteConfigPath():
- return SITE_CONFIG_PATH
diff --git a/infra/scripts/legacy/site_config/config_bootstrap.py b/infra/scripts/legacy/site_config/config_bootstrap.py
deleted file mode 100644
index 8777cab..0000000
--- a/infra/scripts/legacy/site_config/config_bootstrap.py
+++ /dev/null
@@ -1,128 +0,0 @@
-# Copyright (c) 2012 The Chromium Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-"""
-Site configuration information that is sufficient to configure a slave,
-without loading any buildbot or twisted code.
-"""
-
-import inspect
-import os
-
-BASE_DIR = os.path.dirname(os.path.abspath(__file__))
-
-# Override config_default with a config_private file.
-BASE_MASTERS = []
-try:
- import config_private # pylint: disable=F0401
- BASE_MASTERS += [config_private.Master, config_private.PublicMaster]
-except ImportError:
- import config_default as config_private # pylint: disable=W0403
- BASE_MASTERS += [config_private.Master,]
-
-
-class Master(config_private.Master):
- """Buildbot master configuration options."""
-
- trunk_url = (config_private.Master.server_url +
- config_private.Master.repo_root + '/trunk')
-
- webkit_trunk_url = (config_private.Master.webkit_root_url + '/trunk')
-
- trunk_url_src = config_private.Master.git_server_url + '/chromium/src.git'
- trunk_url_tools = trunk_url + '/tools'
- nacl_url = config_private.Master.nacl_trunk_url + '/src/native_client'
- nacl_sdk_root_url = 'https://nativeclient-sdk.googlecode.com/svn'
- nacl_ports_trunk_url = 'https://naclports.googlecode.com/svn/trunk'
- nacl_ports_url = nacl_ports_trunk_url + '/src'
- gears_url = 'http://gears.googlecode.com/svn/trunk'
- gyp_trunk_url = 'http://gyp.googlecode.com/svn/trunk'
- branch_url = (config_private.Master.server_url +
- config_private.Master.repo_root + '/branches')
- merge_branch_url = branch_url + '/chrome_webkit_merge_branch'
- merge_branch_url_src = merge_branch_url + '/src'
-
- v8_url = 'http://v8.googlecode.com/svn'
- v8_branch_url = (v8_url + '/branches')
- v8_bleeding_edge = v8_branch_url + '/bleeding_edge'
- v8_trunk = v8_url + '/trunk'
- es5conform_root_url = "https://es5conform.svn.codeplex.com/svn/"
- es5conform_revision = 62998
-
- dart_url = config_private.Master.googlecode_url % 'dart'
- dart_bleeding = dart_url + '/branches/bleeding_edge'
- dart_trunk = dart_url + '/trunk'
-
- oilpan_url = (config_private.Master.webkit_root_url + '/branches/oilpan')
-
- skia_url = 'http://skia.googlecode.com/svn/'
-
- syzygy_url = 'http://sawbuck.googlecode.com/svn/'
-
- webrtc_url = config_private.Master.googlecode_url % 'webrtc'
- libyuv_url = 'http://libyuv.googlecode.com/svn'
-
- # Default target platform if none was given to the factory.
- default_platform = 'win32'
-
- # Used by the waterfall display.
- project_url = 'http://www.chromium.org'
-
- # Base URL for perf test results.
- perf_base_url = 'http://build.chromium.org/f/chromium/perf'
-
- # Suffix for perf URL.
- perf_report_url_suffix = 'report.html?history=150'
-
- # Directory in which to save perf-test output data files.
- perf_output_dir = '~/www/perf'
-
- # URL pointing to builds and test results.
- archive_url = 'http://build.chromium.org/buildbot'
-
- # The test results server to upload our test results.
- test_results_server = 'test-results.appspot.com'
-
- # File in which to save a list of graph names.
- perf_graph_list = 'graphs.dat'
-
- # Magic step return code inidicating "warning(s)" rather than "error".
- retcode_warnings = 88
-
- @staticmethod
- def GetBotPassword():
- """Returns the slave password retrieved from a local file, or None.
-
- The slave password is loaded from a local file next to this module file, if
- it exists. This is a function rather than a variable so it's not called
- when it's not needed.
-
- We can't both make this a property and also keep it static unless we use a
- <metaclass, which is overkill for this usage.
- """
- # Note: could be overriden by config_private.
- if not getattr(Master, 'bot_password', None):
- # If the bot_password has been requested, the file is required to exist
- # if not overriden in config_private.
- bot_password_path = os.path.join(BASE_DIR, '.bot_password')
- Master.bot_password = open(bot_password_path).read().strip('\n\r')
- return Master.bot_password
-
- @staticmethod
- def _extract_masters(master):
- return [v for v in master.__dict__.itervalues()
- if (inspect.isclass(v) and
- issubclass(v, config_private.Master.Base) and
- v != config_private.Master.Base)]
-
- @classmethod
- def get_base_masters(cls):
- masters = []
- for base_master in BASE_MASTERS:
- masters += cls._extract_masters(base_master)
- return masters
-
- @classmethod
- def get_all_masters(cls):
- return cls._extract_masters(cls)
diff --git a/infra/scripts/legacy/site_config/config_default.py b/infra/scripts/legacy/site_config/config_default.py
deleted file mode 100644
index 8d0cb0a..0000000
--- a/infra/scripts/legacy/site_config/config_default.py
+++ /dev/null
@@ -1,230 +0,0 @@
-# Copyright 2012 The Chromium Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-"""Seeds a number of variables defined in chromium_config.py.
-
-The recommended way is to fork this file and use a custom DEPS forked from
-config/XXX/DEPS with the right configuration data."""
-
-import os
-import re
-import socket
-
-
-SERVICE_ACCOUNTS_PATH = '/creds/service_accounts'
-
-
-class classproperty(object):
- """A decorator that allows is_production_host to only to be defined once."""
- def __init__(self, getter):
- self.getter = getter
- def __get__(self, instance, owner):
- return self.getter(owner)
-
-
-class Master(object):
- # Repository URLs used by the SVNPoller and 'gclient config'.
- server_url = 'http://src.chromium.org'
- repo_root = '/svn'
- git_server_url = 'https://chromium.googlesource.com'
-
- # External repos.
- googlecode_url = 'http://%s.googlecode.com/svn'
- sourceforge_url = 'https://svn.code.sf.net/p/%(repo)s/code'
- googlecode_revlinktmpl = 'https://code.google.com/p/%s/source/browse?r=%s'
-
- # Directly fetches from anonymous Blink svn server.
- webkit_root_url = 'http://src.chromium.org/blink'
- nacl_trunk_url = 'http://src.chromium.org/native_client/trunk'
-
- llvm_url = 'http://llvm.org/svn/llvm-project'
-
- # Perf Dashboard upload URL.
- dashboard_upload_url = 'https://chromeperf.appspot.com'
-
- # Actually for Chromium OS slaves.
- chromeos_url = git_server_url + '/chromiumos.git'
-
- # Default domain for emails to come from and
- # domains to which emails can be sent.
- master_domain = 'example.com'
- permitted_domains = ('example.com',)
-
- # Your smtp server to enable mail notifications.
- smtp = 'smtp'
-
- # By default, bot_password will be filled in by config.GetBotPassword().
- bot_password = None
-
- # Fake urls to make various factories happy.
- trunk_internal_url = None
- trunk_internal_url_src = None
- slave_internal_url = None
- git_internal_server_url = None
- syzygy_internal_url = None
- v8_internal_url = None
-
-
- class Base(object):
- """Master base template.
- Contains stubs for variables that all masters must define."""
-
- # Base service offset for 'master_port'
- MASTER_PORT = 2
- # Base service offset for 'slave_port'
- SLAVE_PORT = 3
- # Base service offset for 'master_port_alt'
- MASTER_PORT_ALT = 4
- # Base service offset for 'try_job_port'
- TRY_JOB_PORT = 5
-
- # A BuildBucket bucket to poll.
- buildbucket_bucket = None
-
- # Master address. You should probably copy this file in another svn repo
- # so you can override this value on both the slaves and the master.
- master_host = 'localhost'
- @classproperty
- def current_host(cls):
- return socket.getfqdn()
- @classproperty
- def in_production(cls):
- return re.match(r'master.*\.golo\.chromium\.org', cls.current_host)
- # Only report that we are running on a master if the master_host (even when
- # master_host is overridden by a subclass) is the same as the current host.
- @classproperty
- def is_production_host(cls):
- return cls.current_host == cls.master_host
-
- # 'from:' field for emails sent from the server.
- from_address = 'nobody@example.com'
- # Additional email addresses to send gatekeeper (automatic tree closage)
- # notifications. Unnecessary for experimental masters and try servers.
- tree_closing_notification_recipients = []
-
- @classproperty
- def master_port(cls):
- return cls._compose_port(cls.MASTER_PORT)
-
- @classproperty
- def slave_port(cls):
- # Which port slaves use to connect to the master.
- return cls._compose_port(cls.SLAVE_PORT)
-
- @classproperty
- def master_port_alt(cls):
- # The alternate read-only page. Optional.
- return cls._compose_port(cls.MASTER_PORT_ALT)
-
- @classproperty
- def try_job_port(cls):
- return cls._compose_port(cls.TRY_JOB_PORT)
-
- @classmethod
- def _compose_port(cls, service):
- """Returns: The port number for 'service' from the master's static config.
-
- Port numbers are mapped of the form:
- XYYZZ
- || \__The last two digits identify the master, e.g. master.chromium
- |\____The second and third digits identify the master host, e.g.
- | master1.golo
- \_____The first digit identifies the port type, e.g. master_port
-
- If any configuration is missing (incremental migration), this method will
- return '0' for that query, indicating no port.
- """
- return (
- (service * 10000) + # X
- (cls.master_port_base * 100) + # YY
- cls.master_port_id) # ZZ
-
- service_account_file = None
-
- @classproperty
- def service_account_path(cls):
- if cls.service_account_file is None:
- return None
- return os.path.join(SERVICE_ACCOUNTS_PATH, cls.service_account_file)
-
- ## Per-master configs.
-
- class Master1(Base):
- """Chromium master."""
- master_host = 'master1.golo.chromium.org'
- master_port_base = 1
- from_address = 'buildbot@chromium.org'
- tree_closing_notification_recipients = [
- 'chromium-build-failure@chromium-gatekeeper-sentry.appspotmail.com']
- base_app_url = 'https://chromium-status.appspot.com'
- tree_status_url = base_app_url + '/status'
- store_revisions_url = base_app_url + '/revisions'
- last_good_url = base_app_url + '/lkgr'
- last_good_blink_url = 'http://blink-status.appspot.com/lkgr'
-
- class Master2(Base):
- """Legacy ChromeOS master."""
- master_host = 'master2.golo.chromium.org'
- master_port_base = 2
- tree_closing_notification_recipients = [
- 'chromeos-build-failures@google.com']
- from_address = 'buildbot@chromium.org'
-
- class Master2a(Base):
- """Chromeos master."""
- master_host = 'master2a.golo.chromium.org'
- master_port_base = 15
- tree_closing_notification_recipients = [
- 'chromeos-build-failures@google.com']
- from_address = 'buildbot@chromium.org'
-
- class Master3(Base):
- """Client master."""
- master_host = 'master3.golo.chromium.org'
- master_port_base = 3
- tree_closing_notification_recipients = []
- from_address = 'buildbot@chromium.org'
-
- class Master4(Base):
- """Try server master."""
- master_host = 'master4.golo.chromium.org'
- master_port_base = 4
- tree_closing_notification_recipients = []
- from_address = 'tryserver@chromium.org'
- code_review_site = 'https://codereview.chromium.org'
-
- class Master4a(Base):
- """Try server master."""
- master_host = 'master4a.golo.chromium.org'
- master_port_base = 14
- tree_closing_notification_recipients = []
- from_address = 'tryserver@chromium.org'
- code_review_site = 'https://codereview.chromium.org'
-
- ## Native Client related
-
- class NaClBase(Master3):
- """Base class for Native Client masters."""
- tree_closing_notification_recipients = ['bradnelson@chromium.org']
- base_app_url = 'https://nativeclient-status.appspot.com'
- tree_status_url = base_app_url + '/status'
- store_revisions_url = base_app_url + '/revisions'
- last_good_url = base_app_url + '/lkgr'
- perf_base_url = 'http://build.chromium.org/f/client/perf'
-
- ## ChromiumOS related
-
- class ChromiumOSBase(Master2):
- """Legacy base class for ChromiumOS masters"""
- base_app_url = 'https://chromiumos-status.appspot.com'
- tree_status_url = base_app_url + '/status'
- store_revisions_url = base_app_url + '/revisions'
- last_good_url = base_app_url + '/lkgr'
-
- class ChromiumOSBase2a(Master2a):
- """Base class for ChromiumOS masters"""
- base_app_url = 'https://chromiumos-status.appspot.com'
- tree_status_url = base_app_url + '/status'
- store_revisions_url = base_app_url + '/revisions'
- last_good_url = base_app_url + '/lkgr'