summaryrefslogtreecommitdiffstats
path: root/tools/auto_bisect
diff options
context:
space:
mode:
authorqyearsley@chromium.org <qyearsley@chromium.org@0039d316-1c4b-4281-b951-d872f2087c98>2014-07-10 01:27:09 +0000
committerqyearsley@chromium.org <qyearsley@chromium.org@0039d316-1c4b-4281-b951-d872f2087c98>2014-07-10 01:27:09 +0000
commitd4aad20e68c96a09f8269e0aa19bf2583d8a7331 (patch)
treed7abf0c00861174df0943d00b0512061218715bf /tools/auto_bisect
parent927c0bf8213ccff52ae6e2eb30b033bf5517497f (diff)
downloadchromium_src-d4aad20e68c96a09f8269e0aa19bf2583d8a7331.zip
chromium_src-d4aad20e68c96a09f8269e0aa19bf2583d8a7331.tar.gz
chromium_src-d4aad20e68c96a09f8269e0aa19bf2583d8a7331.tar.bz2
Make a directory in which to put bisect-related modules.
BUG= Review URL: https://codereview.chromium.org/359013002 git-svn-id: svn://svn.chromium.org/chrome/trunk/src@282209 0039d316-1c4b-4281-b951-d872f2087c98
Diffstat (limited to 'tools/auto_bisect')
-rw-r--r--tools/auto_bisect/OWNERS4
-rw-r--r--tools/auto_bisect/README23
-rw-r--r--tools/auto_bisect/__init__.py0
-rw-r--r--tools/auto_bisect/bisect_utils.py510
-rw-r--r--tools/auto_bisect/post_perf_builder_job.py372
5 files changed, 909 insertions, 0 deletions
diff --git a/tools/auto_bisect/OWNERS b/tools/auto_bisect/OWNERS
new file mode 100644
index 0000000..a450d85
--- /dev/null
+++ b/tools/auto_bisect/OWNERS
@@ -0,0 +1,4 @@
+prasadv@chromium.org
+qyearsley@chromium.org
+simonhatch@chromium.org
+tonyg@chromium.org
diff --git a/tools/auto_bisect/README b/tools/auto_bisect/README
new file mode 100644
index 0000000..d43222c
--- /dev/null
+++ b/tools/auto_bisect/README
@@ -0,0 +1,23 @@
+This directory contains modules related to tools for bisecting regressions.
+
+There are several different tools for bisecting regressions; the main use
+of these tools is to find revisions where a performance regression occurred.
+These tools are generally run by trybots but can also be run locally.
+
+Documentation:
+ http://www.chromium.org/developers/bisecting-bugs
+ http://www.chromium.org/developers/tree-sheriffs/perf-sheriffs/bisecting-performance-regressions
+
+Overview of bisect-related files in src/tools:
+ run-bisect-perf-regression.py
+ -- the script used to kick off a normal performance regression bisect job.
+ run-bisect-perf-regression.cfg
+ -- this file contains parameters for a bisect job, and is read by other
+ modules including run-bisect-perf-regresion.py.
+ run-bisect-manual-test.py
+ -- a script which is used to manually bisect regressions; this also
+ depends on bisect-perf-gression.py.
+ bisect-perf-regression.py
+ -- the main module which the others depend on.
+ bisect-manual-test.py
+ -- a helper module used when manually bisect regressions.
diff --git a/tools/auto_bisect/__init__.py b/tools/auto_bisect/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/tools/auto_bisect/__init__.py
diff --git a/tools/auto_bisect/bisect_utils.py b/tools/auto_bisect/bisect_utils.py
new file mode 100644
index 0000000..7dc9a9b
--- /dev/null
+++ b/tools/auto_bisect/bisect_utils.py
@@ -0,0 +1,510 @@
+# Copyright 2014 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Set of operations/utilities related to checking out the depot, and
+outputting annotations on the buildbot waterfall. These are intended to be
+used by the bisection scripts."""
+
+import errno
+import imp
+import os
+import shutil
+import stat
+import subprocess
+import sys
+
+DEFAULT_GCLIENT_CUSTOM_DEPS = {
+ "src/data/page_cycler": "https://chrome-internal.googlesource.com/"
+ "chrome/data/page_cycler/.git",
+ "src/data/dom_perf": "https://chrome-internal.googlesource.com/"
+ "chrome/data/dom_perf/.git",
+ "src/data/mach_ports": "https://chrome-internal.googlesource.com/"
+ "chrome/data/mach_ports/.git",
+ "src/tools/perf/data": "https://chrome-internal.googlesource.com/"
+ "chrome/tools/perf/data/.git",
+ "src/third_party/adobe/flash/binaries/ppapi/linux":
+ "https://chrome-internal.googlesource.com/"
+ "chrome/deps/adobe/flash/binaries/ppapi/linux/.git",
+ "src/third_party/adobe/flash/binaries/ppapi/linux_x64":
+ "https://chrome-internal.googlesource.com/"
+ "chrome/deps/adobe/flash/binaries/ppapi/linux_x64/.git",
+ "src/third_party/adobe/flash/binaries/ppapi/mac":
+ "https://chrome-internal.googlesource.com/"
+ "chrome/deps/adobe/flash/binaries/ppapi/mac/.git",
+ "src/third_party/adobe/flash/binaries/ppapi/mac_64":
+ "https://chrome-internal.googlesource.com/"
+ "chrome/deps/adobe/flash/binaries/ppapi/mac_64/.git",
+ "src/third_party/adobe/flash/binaries/ppapi/win":
+ "https://chrome-internal.googlesource.com/"
+ "chrome/deps/adobe/flash/binaries/ppapi/win/.git",
+ "src/third_party/adobe/flash/binaries/ppapi/win_x64":
+ "https://chrome-internal.googlesource.com/"
+ "chrome/deps/adobe/flash/binaries/ppapi/win_x64/.git",
+ "src/chrome/tools/test/reference_build/chrome_win": None,
+ "src/chrome/tools/test/reference_build/chrome_mac": None,
+ "src/chrome/tools/test/reference_build/chrome_linux": None,
+ "src/third_party/WebKit/LayoutTests": None,
+ "src/tools/valgrind": None,}
+
+GCLIENT_SPEC_DATA = [
+ { "name" : "src",
+ "url" : "https://chromium.googlesource.com/chromium/src.git",
+ "deps_file" : ".DEPS.git",
+ "managed" : True,
+ "custom_deps" : {},
+ "safesync_url": "",
+ },
+]
+GCLIENT_SPEC_ANDROID = "\ntarget_os = ['android']"
+GCLIENT_CUSTOM_DEPS_V8 = {"src/v8_bleeding_edge": "git://github.com/v8/v8.git"}
+FILE_DEPS_GIT = '.DEPS.git'
+FILE_DEPS = 'DEPS'
+
+REPO_PARAMS = [
+ 'https://chrome-internal.googlesource.com/chromeos/manifest-internal/',
+ '--repo-url',
+ 'https://git.chromium.org/external/repo.git'
+]
+
+REPO_SYNC_COMMAND = 'git checkout -f $(git rev-list --max-count=1 '\
+ '--before=%d remotes/m/master)'
+
+ORIGINAL_ENV = {}
+
+def OutputAnnotationStepStart(name):
+ """Outputs appropriate annotation to signal the start of a step to
+ a trybot.
+
+ Args:
+ name: The name of the step.
+ """
+ print
+ print '@@@SEED_STEP %s@@@' % name
+ print '@@@STEP_CURSOR %s@@@' % name
+ print '@@@STEP_STARTED@@@'
+ print
+ sys.stdout.flush()
+
+
+def OutputAnnotationStepClosed():
+ """Outputs appropriate annotation to signal the closing of a step to
+ a trybot."""
+ print
+ print '@@@STEP_CLOSED@@@'
+ print
+ sys.stdout.flush()
+
+
+def OutputAnnotationStepLink(label, url):
+ """Outputs appropriate annotation to print a link.
+
+ Args:
+ label: The name to print.
+ url: The url to print.
+ """
+ print
+ print '@@@STEP_LINK@%s@%s@@@' % (label, url)
+ print
+ sys.stdout.flush()
+
+
+def LoadExtraSrc(path_to_file):
+ """Attempts to load an extra source file. If this is successful, uses the
+ new module to override some global values, such as gclient spec data.
+
+ Returns:
+ The loaded src module, or None."""
+ try:
+ global GCLIENT_SPEC_DATA
+ global GCLIENT_SPEC_ANDROID
+ extra_src = imp.load_source('data', path_to_file)
+ GCLIENT_SPEC_DATA = extra_src.GetGClientSpec()
+ GCLIENT_SPEC_ANDROID = extra_src.GetGClientSpecExtraParams()
+ return extra_src
+ except ImportError, e:
+ return None
+
+
+def IsTelemetryCommand(command):
+ """Attempts to discern whether or not a given command is running telemetry."""
+ return ('tools/perf/run_' in command or 'tools\\perf\\run_' in command)
+
+
+def CreateAndChangeToSourceDirectory(working_directory):
+ """Creates a directory 'bisect' as a subdirectory of 'working_directory'. If
+ the function is successful, the current working directory will change to that
+ of the new 'bisect' directory.
+
+ Returns:
+ True if the directory was successfully created (or already existed).
+ """
+ cwd = os.getcwd()
+ os.chdir(working_directory)
+ try:
+ os.mkdir('bisect')
+ except OSError, e:
+ if e.errno != errno.EEXIST:
+ return False
+ os.chdir('bisect')
+ return True
+
+
+def SubprocessCall(cmd, cwd=None):
+ """Runs a subprocess with specified parameters.
+
+ Args:
+ params: A list of parameters to pass to gclient.
+ cwd: Working directory to run from.
+
+ Returns:
+ The return code of the call.
+ """
+ if os.name == 'nt':
+ # "HOME" isn't normally defined on windows, but is needed
+ # for git to find the user's .netrc file.
+ if not os.getenv('HOME'):
+ os.environ['HOME'] = os.environ['USERPROFILE']
+ shell = os.name == 'nt'
+ return subprocess.call(cmd, shell=shell, cwd=cwd)
+
+
+def RunGClient(params, cwd=None):
+ """Runs gclient with the specified parameters.
+
+ Args:
+ params: A list of parameters to pass to gclient.
+ cwd: Working directory to run from.
+
+ Returns:
+ The return code of the call.
+ """
+ cmd = ['gclient'] + params
+
+ return SubprocessCall(cmd, cwd=cwd)
+
+
+def RunRepo(params):
+ """Runs cros repo command with specified parameters.
+
+ Args:
+ params: A list of parameters to pass to gclient.
+
+ Returns:
+ The return code of the call.
+ """
+ cmd = ['repo'] + params
+
+ return SubprocessCall(cmd)
+
+
+def RunRepoSyncAtTimestamp(timestamp):
+ """Syncs all git depots to the timestamp specified using repo forall.
+
+ Args:
+ params: Unix timestamp to sync to.
+
+ Returns:
+ The return code of the call.
+ """
+ repo_sync = REPO_SYNC_COMMAND % timestamp
+ cmd = ['forall', '-c', REPO_SYNC_COMMAND % timestamp]
+ return RunRepo(cmd)
+
+
+def RunGClientAndCreateConfig(opts, custom_deps=None, cwd=None):
+ """Runs gclient and creates a config containing both src and src-internal.
+
+ Args:
+ opts: The options parsed from the command line through parse_args().
+ custom_deps: A dictionary of additional dependencies to add to .gclient.
+ cwd: Working directory to run from.
+
+ Returns:
+ The return code of the call.
+ """
+ spec = GCLIENT_SPEC_DATA
+
+ if custom_deps:
+ for k, v in custom_deps.iteritems():
+ spec[0]['custom_deps'][k] = v
+
+ # Cannot have newlines in string on windows
+ spec = 'solutions =' + str(spec)
+ spec = ''.join([l for l in spec.splitlines()])
+
+ if 'android' in opts.target_platform:
+ spec += GCLIENT_SPEC_ANDROID
+
+ return_code = RunGClient(
+ ['config', '--spec=%s' % spec, '--git-deps'], cwd=cwd)
+ return return_code
+
+
+def IsDepsFileBlink():
+ """Reads .DEPS.git and returns whether or not we're using blink.
+
+ Returns:
+ True if blink, false if webkit.
+ """
+ locals = {'Var': lambda _: locals["vars"][_],
+ 'From': lambda *args: None}
+ execfile(FILE_DEPS_GIT, {}, locals)
+ return 'blink.git' in locals['vars']['webkit_url']
+
+
+def OnAccessError(func, path, exc_info):
+ """
+ Source: http://stackoverflow.com/questions/2656322/python-shutil-rmtree-fails-on-windows-with-access-is-denied
+
+ Error handler for ``shutil.rmtree``.
+
+ If the error is due to an access error (read only file)
+ it attempts to add write permission and then retries.
+
+ If the error is for another reason it re-raises the error.
+
+ Args:
+ func: The function that raised the error.
+ path: The path name passed to func.
+ exc_info: Exception information returned by sys.exc_info().
+ """
+ if not os.access(path, os.W_OK):
+ # Is the error an access error ?
+ os.chmod(path, stat.S_IWUSR)
+ func(path)
+ else:
+ raise
+
+
+def RemoveThirdPartyDirectory(dir_name):
+ """Removes third_party directory from the source.
+
+ At some point, some of the third_parties were causing issues to changes in
+ the way they are synced. We remove such folder in order to avoid sync errors
+ while bisecting.
+
+ Returns:
+ True on success, otherwise False.
+ """
+ path_to_dir = os.path.join(os.getcwd(), 'third_party', dir_name)
+ try:
+ if os.path.exists(path_to_dir):
+ shutil.rmtree(path_to_dir, onerror=OnAccessError)
+ except OSError, e:
+ print 'Error #%d while running shutil.rmtree(%s): %s' % (
+ e.errno, path_to_dir, str(e))
+ if e.errno != errno.ENOENT:
+ return False
+ return True
+
+
+def _CleanupPreviousGitRuns():
+ """Performs necessary cleanup between runs."""
+ # If a previous run of git crashed, bot was reset, etc... we
+ # might end up with leftover index.lock files.
+ for (path, dir, files) in os.walk(os.getcwd()):
+ for cur_file in files:
+ if cur_file.endswith('index.lock'):
+ path_to_file = os.path.join(path, cur_file)
+ os.remove(path_to_file)
+
+
+def RunGClientAndSync(cwd=None):
+ """Runs gclient and does a normal sync.
+
+ Args:
+ cwd: Working directory to run from.
+
+ Returns:
+ The return code of the call.
+ """
+ params = ['sync', '--verbose', '--nohooks', '--reset', '--force']
+ return RunGClient(params, cwd=cwd)
+
+
+def SetupGitDepot(opts, custom_deps):
+ """Sets up the depot for the bisection. The depot will be located in a
+ subdirectory called 'bisect'.
+
+ Args:
+ opts: The options parsed from the command line through parse_args().
+ custom_deps: A dictionary of additional dependencies to add to .gclient.
+
+ Returns:
+ True if gclient successfully created the config file and did a sync, False
+ otherwise.
+ """
+ name = 'Setting up Bisection Depot'
+
+ if opts.output_buildbot_annotations:
+ OutputAnnotationStepStart(name)
+
+ passed = False
+
+ if not RunGClientAndCreateConfig(opts, custom_deps):
+ passed_deps_check = True
+ if os.path.isfile(os.path.join('src', FILE_DEPS_GIT)):
+ cwd = os.getcwd()
+ os.chdir('src')
+ if not IsDepsFileBlink():
+ passed_deps_check = RemoveThirdPartyDirectory('Webkit')
+ else:
+ passed_deps_check = True
+ if passed_deps_check:
+ passed_deps_check = RemoveThirdPartyDirectory('libjingle')
+ if passed_deps_check:
+ passed_deps_check = RemoveThirdPartyDirectory('skia')
+ os.chdir(cwd)
+
+ if passed_deps_check:
+ _CleanupPreviousGitRuns()
+
+ RunGClient(['revert'])
+ if not RunGClientAndSync():
+ passed = True
+
+ if opts.output_buildbot_annotations:
+ print
+ OutputAnnotationStepClosed()
+
+ return passed
+
+
+def SetupCrosRepo():
+ """Sets up cros repo for bisecting chromeos.
+
+ Returns:
+ Returns 0 on success.
+ """
+ cwd = os.getcwd()
+ try:
+ os.mkdir('cros')
+ except OSError, e:
+ if e.errno != errno.EEXIST:
+ return False
+ os.chdir('cros')
+
+ cmd = ['init', '-u'] + REPO_PARAMS
+
+ passed = False
+
+ if not RunRepo(cmd):
+ if not RunRepo(['sync']):
+ passed = True
+ os.chdir(cwd)
+
+ return passed
+
+
+def CopyAndSaveOriginalEnvironmentVars():
+ """Makes a copy of the current environment variables."""
+ # TODO: Waiting on crbug.com/255689, will remove this after.
+ vars_to_remove = []
+ for k, v in os.environ.iteritems():
+ if 'ANDROID' in k:
+ vars_to_remove.append(k)
+ vars_to_remove.append('CHROME_SRC')
+ vars_to_remove.append('CHROMIUM_GYP_FILE')
+ vars_to_remove.append('GYP_CROSSCOMPILE')
+ vars_to_remove.append('GYP_DEFINES')
+ vars_to_remove.append('GYP_GENERATORS')
+ vars_to_remove.append('GYP_GENERATOR_FLAGS')
+ vars_to_remove.append('OBJCOPY')
+ for k in vars_to_remove:
+ if os.environ.has_key(k):
+ del os.environ[k]
+
+ global ORIGINAL_ENV
+ ORIGINAL_ENV = os.environ.copy()
+
+
+def SetupAndroidBuildEnvironment(opts, path_to_src=None):
+ """Sets up the android build environment.
+
+ Args:
+ opts: The options parsed from the command line through parse_args().
+ path_to_src: Path to the src checkout.
+
+ Returns:
+ True if successful.
+ """
+
+ # Revert the environment variables back to default before setting them up
+ # with envsetup.sh.
+ env_vars = os.environ.copy()
+ for k, _ in env_vars.iteritems():
+ del os.environ[k]
+ for k, v in ORIGINAL_ENV.iteritems():
+ os.environ[k] = v
+
+ path_to_file = os.path.join('build', 'android', 'envsetup.sh')
+ proc = subprocess.Popen(['bash', '-c', 'source %s && env' % path_to_file],
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ cwd=path_to_src)
+ (out, _) = proc.communicate()
+
+ for line in out.splitlines():
+ (k, _, v) = line.partition('=')
+ os.environ[k] = v
+ # envsetup.sh no longer sets OS=android to GYP_DEFINES env variable
+ # (CL/170273005). Set this variable explicitly inorder to build chrome on
+ # android.
+ try:
+ if 'OS=android' not in os.environ['GYP_DEFINES']:
+ os.environ['GYP_DEFINES'] = '%s %s' % (os.environ['GYP_DEFINES'],
+ 'OS=android')
+ except KeyError:
+ os.environ['GYP_DEFINES'] = 'OS=android'
+
+ if opts.use_goma:
+ os.environ['GYP_DEFINES'] = '%s %s' % (os.environ['GYP_DEFINES'],
+ 'use_goma=1')
+ return not proc.returncode
+
+
+def SetupPlatformBuildEnvironment(opts):
+ """Performs any platform specific setup.
+
+ Args:
+ opts: The options parsed from the command line through parse_args().
+
+ Returns:
+ True if successful.
+ """
+ if 'android' in opts.target_platform:
+ CopyAndSaveOriginalEnvironmentVars()
+ return SetupAndroidBuildEnvironment(opts)
+ elif opts.target_platform == 'cros':
+ return SetupCrosRepo()
+
+ return True
+
+
+def CheckIfBisectDepotExists(opts):
+ """Checks if the bisect directory already exists.
+
+ Args:
+ opts: The options parsed from the command line through parse_args().
+
+ Returns:
+ Returns True if it exists.
+ """
+ path_to_dir = os.path.join(opts.working_directory, 'bisect', 'src')
+ return os.path.exists(path_to_dir)
+
+
+def CreateBisectDirectoryAndSetupDepot(opts, custom_deps):
+ """Sets up a subdirectory 'bisect' and then retrieves a copy of the depot
+ there using gclient.
+
+ Args:
+ opts: The options parsed from the command line through parse_args().
+ custom_deps: A dictionary of additional dependencies to add to .gclient.
+ """
+ if not CreateAndChangeToSourceDirectory(opts.working_directory):
+ raise RuntimeError('Could not create bisect directory.')
+
+ if not SetupGitDepot(opts, custom_deps):
+ raise RuntimeError('Failed to grab source.')
diff --git a/tools/auto_bisect/post_perf_builder_job.py b/tools/auto_bisect/post_perf_builder_job.py
new file mode 100644
index 0000000..97c6ecc
--- /dev/null
+++ b/tools/auto_bisect/post_perf_builder_job.py
@@ -0,0 +1,372 @@
+# Copyright 2014 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Post a try job request via HTTP to the Tryserver to produce build."""
+
+import getpass
+import json
+import optparse
+import os
+import sys
+import urllib
+import urllib2
+
+# Link to get JSON data of builds
+BUILDER_JSON_URL = ('%(server_url)s/json/builders/%(bot_name)s/builds/'
+ '%(build_num)s?as_text=1&filter=0')
+
+# Link to display build steps
+BUILDER_HTML_URL = ('%(server_url)s/builders/%(bot_name)s/builds/%(build_num)s')
+
+# Tryserver buildbots status page
+TRY_SERVER_URL = 'http://build.chromium.org/p/tryserver.chromium.perf'
+
+# Hostname of the tryserver where perf bisect builders are hosted. This is used
+# for posting build request to tryserver.
+BISECT_BUILDER_HOST = 'master4.golo.chromium.org'
+# 'try_job_port' on tryserver to post build request.
+BISECT_BUILDER_PORT = 8341
+
+
+# From buildbot.status.builder.
+# See: http://docs.buildbot.net/current/developer/results.html
+SUCCESS, WARNINGS, FAILURE, SKIPPED, EXCEPTION, RETRY, TRYPENDING = range(7)
+
+# Status codes that can be returned by the GetBuildStatus method.
+OK = (SUCCESS, WARNINGS)
+# Indicates build failure.
+FAILED = (FAILURE, EXCEPTION, SKIPPED)
+# Inidcates build in progress or in pending queue.
+PENDING = (RETRY, TRYPENDING)
+
+
+class ServerAccessError(Exception):
+
+ def __str__(self):
+ return '%s\nSorry, cannot connect to server.' % self.args[0]
+
+
+def PostTryJob(url_params):
+ """Sends a build request to the server using the HTTP protocol.
+
+ Args:
+ url_params: A dictionary of query parameters to be sent in the request.
+ In order to post build request to try server, this dictionary
+ should contain information for following keys:
+ 'host': Hostname of the try server.
+ 'port': Port of the try server.
+ 'revision': SVN Revision to build.
+ 'bot': Name of builder bot which would be used.
+ Returns:
+ True if the request is posted successfully. Otherwise throws an exception.
+ """
+ # Parse url parameters to be sent to Try server.
+ if not url_params.get('host'):
+ raise ValueError('Hostname of server to connect is missing.')
+ if not url_params.get('port'):
+ raise ValueError('Port of server to connect is missing.')
+ if not url_params.get('revision'):
+ raise ValueError('Missing revision details. Please specify revision'
+ ' information.')
+ if not url_params.get('bot'):
+ raise ValueError('Missing bot details. Please specify bot information.')
+
+ # Pop 'host' and 'port' to avoid passing them as query params.
+ url = 'http://%s:%s/send_try_patch' % (url_params.pop('host'),
+ url_params.pop('port'))
+
+ print 'Sending by HTTP'
+ query_params = '&'.join('%s=%s' % (k, v) for k, v in url_params.iteritems())
+ print 'url: %s?%s' % (url, query_params)
+
+ connection = None
+ try:
+ print 'Opening connection...'
+ connection = urllib2.urlopen(url, urllib.urlencode(url_params))
+ print 'Done, request sent to server to produce build.'
+ except IOError, e:
+ raise ServerAccessError('%s is unaccessible. Reason: %s' % (url, e))
+ if not connection:
+ raise ServerAccessError('%s is unaccessible.' % url)
+ response = connection.read()
+ print 'Received %s from server' % response
+ if response != 'OK':
+ raise ServerAccessError('%s is unaccessible. Got:\n%s' % (url, response))
+ return True
+
+
+def _IsBuildRunning(build_data):
+ """Checks whether the build is in progress on buildbot.
+
+ Presence of currentStep element in build JSON indicates build is in progress.
+
+ Args:
+ build_data: A dictionary with build data, loaded from buildbot JSON API.
+
+ Returns:
+ True if build is in progress, otherwise False.
+ """
+ current_step = build_data.get('currentStep')
+ if (current_step and current_step.get('isStarted') and
+ current_step.get('results') is None):
+ return True
+ return False
+
+
+def _IsBuildFailed(build_data):
+ """Checks whether the build failed on buildbot.
+
+ Sometime build status is marked as failed even though compile and packaging
+ steps are successful. This may happen due to some intermediate steps of less
+ importance such as gclient revert, generate_telemetry_profile are failed.
+ Therefore we do an addition check to confirm if build was successful by
+ calling _IsBuildSuccessful.
+
+ Args:
+ build_data: A dictionary with build data, loaded from buildbot JSON API.
+
+ Returns:
+ True if revision is failed build, otherwise False.
+ """
+ if (build_data.get('results') in FAILED and
+ not _IsBuildSuccessful(build_data)):
+ return True
+ return False
+
+
+def _IsBuildSuccessful(build_data):
+ """Checks whether the build succeeded on buildbot.
+
+ We treat build as successful if the package_build step is completed without
+ any error i.e., when results attribute of the this step has value 0 or 1
+ in its first element.
+
+ Args:
+ build_data: A dictionary with build data, loaded from buildbot JSON API.
+
+ Returns:
+ True if revision is successfully build, otherwise False.
+ """
+ if build_data.get('steps'):
+ for item in build_data.get('steps'):
+ # The 'results' attribute of each step consists of two elements,
+ # results[0]: This represents the status of build step.
+ # See: http://docs.buildbot.net/current/developer/results.html
+ # results[1]: List of items, contains text if step fails, otherwise empty.
+ if (item.get('name') == 'package_build' and
+ item.get('isFinished') and
+ item.get('results')[0] in OK):
+ return True
+ return False
+
+
+def _FetchBuilderData(builder_url):
+ """Fetches JSON data for the all the builds from the tryserver.
+
+ Args:
+ builder_url: A tryserver URL to fetch builds information.
+
+ Returns:
+ A dictionary with information of all build on the tryserver.
+ """
+ data = None
+ try:
+ url = urllib2.urlopen(builder_url)
+ except urllib2.URLError, e:
+ print ('urllib2.urlopen error %s, waterfall status page down.[%s]' % (
+ builder_url, str(e)))
+ return None
+ if url is not None:
+ try:
+ data = url.read()
+ except IOError, e:
+ print 'urllib2 file object read error %s, [%s].' % (builder_url, str(e))
+ return data
+
+
+def _GetBuildData(buildbot_url):
+ """Gets build information for the given build id from the tryserver.
+
+ Args:
+ buildbot_url: A tryserver URL to fetch build information.
+
+ Returns:
+ A dictionary with build information if build exists, otherwise None.
+ """
+ builds_json = _FetchBuilderData(buildbot_url)
+ if builds_json:
+ return json.loads(builds_json)
+ return None
+
+
+def _GetBuildBotUrl(builder_host, builder_port):
+ """Gets build bot URL based on the host and port of the builders.
+
+ Note: All bisect builder bots are hosted on tryserver.chromium i.e.,
+ on master4:8328, since we cannot access tryserver using host and port
+ number directly, we use tryserver URL.
+
+ Args:
+ builder_host: Hostname of the server where the builder is hosted.
+ builder_port: Port number of ther server where the builder is hosted.
+
+ Returns:
+ URL of the buildbot as a string.
+ """
+ if (builder_host == BISECT_BUILDER_HOST and
+ builder_port == BISECT_BUILDER_PORT):
+ return TRY_SERVER_URL
+ else:
+ return 'http://%s:%s' % (builder_host, builder_port)
+
+
+def GetBuildStatus(build_num, bot_name, builder_host, builder_port):
+ """Gets build status from the buildbot status page for a given build number.
+
+ Args:
+ build_num: A build number on tryserver to determine its status.
+ bot_name: Name of the bot where the build information is scanned.
+ builder_host: Hostname of the server where the builder is hosted.
+ builder_port: Port number of ther server where the builder is hosted.
+
+ Returns:
+ A tuple consists of build status (SUCCESS, FAILED or PENDING) and a link
+ to build status page on the waterfall.
+ """
+ results_url = None
+ if build_num:
+ # Gets the buildbot url for the given host and port.
+ server_url = _GetBuildBotUrl(builder_host, builder_port)
+ buildbot_url = BUILDER_JSON_URL % {'server_url': server_url,
+ 'bot_name': bot_name,
+ 'build_num': build_num
+ }
+ build_data = _GetBuildData(buildbot_url)
+ if build_data:
+ # Link to build on the buildbot showing status of build steps.
+ results_url = BUILDER_HTML_URL % {'server_url': server_url,
+ 'bot_name': bot_name,
+ 'build_num': build_num
+ }
+ if _IsBuildFailed(build_data):
+ return (FAILED, results_url)
+
+ elif _IsBuildSuccessful(build_data):
+ return (OK, results_url)
+ return (PENDING, results_url)
+
+
+def GetBuildNumFromBuilder(build_reason, bot_name, builder_host, builder_port):
+ """Gets build number on build status page for a given build reason.
+
+ It parses the JSON data from buildbot page and collect basic information
+ about the all the builds and then this uniquely identifies the build based
+ on the 'reason' attribute in builds's JSON data.
+ The 'reason' attribute set while a build request is posted, and same is used
+ to identify the build on status page.
+
+ Args:
+ build_reason: A unique build name set to build on tryserver.
+ bot_name: Name of the bot where the build information is scanned.
+ builder_host: Hostname of the server where the builder is hosted.
+ builder_port: Port number of ther server where the builder is hosted.
+
+ Returns:
+ A build number as a string if found, otherwise None.
+ """
+ # Gets the buildbot url for the given host and port.
+ server_url = _GetBuildBotUrl(builder_host, builder_port)
+ buildbot_url = BUILDER_JSON_URL % {'server_url': server_url,
+ 'bot_name': bot_name,
+ 'build_num': '_all'
+ }
+ builds_json = _FetchBuilderData(buildbot_url)
+ if builds_json:
+ builds_data = json.loads(builds_json)
+ for current_build in builds_data:
+ if builds_data[current_build].get('reason') == build_reason:
+ return builds_data[current_build].get('number')
+ return None
+
+
+def _GetQueryParams(options):
+ """Parses common query parameters which will be passed to PostTryJob.
+
+ Args:
+ options: The options object parsed from the command line.
+
+ Returns:
+ A dictionary consists of query parameters.
+ """
+ values = {'host': options.host,
+ 'port': options.port,
+ 'user': options.user,
+ 'name': options.name
+ }
+ if options.email:
+ values['email'] = options.email
+ if options.revision:
+ values['revision'] = options.revision
+ if options.root:
+ values['root'] = options.root
+ if options.bot:
+ values['bot'] = options.bot
+ if options.patch:
+ values['patch'] = options.patch
+ return values
+
+
+def _GenParser():
+ """Parses the command line for posting build request."""
+ usage = ('%prog [options]\n'
+ 'Post a build request to the try server for the given revision.\n')
+ parser = optparse.OptionParser(usage=usage)
+ parser.add_option('-H', '--host',
+ help='Host address of the try server.')
+ parser.add_option('-P', '--port', type='int',
+ help='HTTP port of the try server.')
+ parser.add_option('-u', '--user', default=getpass.getuser(),
+ dest='user',
+ help='Owner user name [default: %default]')
+ parser.add_option('-e', '--email',
+ default=os.environ.get('TRYBOT_RESULTS_EMAIL_ADDRESS',
+ os.environ.get('EMAIL_ADDRESS')),
+ help=('Email address where to send the results. Use either '
+ 'the TRYBOT_RESULTS_EMAIL_ADDRESS environment '
+ 'variable or EMAIL_ADDRESS to set the email address '
+ 'the try bots report results to [default: %default]'))
+ parser.add_option('-n', '--name',
+ default='try_job_http',
+ help='Descriptive name of the try job')
+ parser.add_option('-b', '--bot',
+ help=('IMPORTANT: specify ONE builder per run is supported.'
+ 'Run script for each builders separately.'))
+ parser.add_option('-r', '--revision',
+ help=('Revision to use for the try job; default: the '
+ 'revision will be determined by the try server; see '
+ 'its waterfall for more info'))
+ parser.add_option('--root',
+ help=('Root to use for the patch; base subdirectory for '
+ 'patch created in a subdirectory'))
+ parser.add_option('--patch',
+ help='Patch information.')
+ return parser
+
+
+def Main(argv):
+ parser = _GenParser()
+ options, _ = parser.parse_args()
+ if not options.host:
+ raise ServerAccessError('Please use the --host option to specify the try '
+ 'server host to connect to.')
+ if not options.port:
+ raise ServerAccessError('Please use the --port option to specify the try '
+ 'server port to connect to.')
+ params = _GetQueryParams(options)
+ PostTryJob(params)
+
+
+if __name__ == '__main__':
+ sys.exit(Main(sys.argv))
+