diff options
author | qyearsley@chromium.org <qyearsley@chromium.org@0039d316-1c4b-4281-b951-d872f2087c98> | 2014-07-10 01:27:09 +0000 |
---|---|---|
committer | qyearsley@chromium.org <qyearsley@chromium.org@0039d316-1c4b-4281-b951-d872f2087c98> | 2014-07-10 01:27:09 +0000 |
commit | d4aad20e68c96a09f8269e0aa19bf2583d8a7331 (patch) | |
tree | d7abf0c00861174df0943d00b0512061218715bf /tools/auto_bisect | |
parent | 927c0bf8213ccff52ae6e2eb30b033bf5517497f (diff) | |
download | chromium_src-d4aad20e68c96a09f8269e0aa19bf2583d8a7331.zip chromium_src-d4aad20e68c96a09f8269e0aa19bf2583d8a7331.tar.gz chromium_src-d4aad20e68c96a09f8269e0aa19bf2583d8a7331.tar.bz2 |
Make a directory in which to put bisect-related modules.
BUG=
Review URL: https://codereview.chromium.org/359013002
git-svn-id: svn://svn.chromium.org/chrome/trunk/src@282209 0039d316-1c4b-4281-b951-d872f2087c98
Diffstat (limited to 'tools/auto_bisect')
-rw-r--r-- | tools/auto_bisect/OWNERS | 4 | ||||
-rw-r--r-- | tools/auto_bisect/README | 23 | ||||
-rw-r--r-- | tools/auto_bisect/__init__.py | 0 | ||||
-rw-r--r-- | tools/auto_bisect/bisect_utils.py | 510 | ||||
-rw-r--r-- | tools/auto_bisect/post_perf_builder_job.py | 372 |
5 files changed, 909 insertions, 0 deletions
diff --git a/tools/auto_bisect/OWNERS b/tools/auto_bisect/OWNERS new file mode 100644 index 0000000..a450d85 --- /dev/null +++ b/tools/auto_bisect/OWNERS @@ -0,0 +1,4 @@ +prasadv@chromium.org +qyearsley@chromium.org +simonhatch@chromium.org +tonyg@chromium.org diff --git a/tools/auto_bisect/README b/tools/auto_bisect/README new file mode 100644 index 0000000..d43222c --- /dev/null +++ b/tools/auto_bisect/README @@ -0,0 +1,23 @@ +This directory contains modules related to tools for bisecting regressions. + +There are several different tools for bisecting regressions; the main use +of these tools is to find revisions where a performance regression occurred. +These tools are generally run by trybots but can also be run locally. + +Documentation: + http://www.chromium.org/developers/bisecting-bugs + http://www.chromium.org/developers/tree-sheriffs/perf-sheriffs/bisecting-performance-regressions + +Overview of bisect-related files in src/tools: + run-bisect-perf-regression.py + -- the script used to kick off a normal performance regression bisect job. + run-bisect-perf-regression.cfg + -- this file contains parameters for a bisect job, and is read by other + modules including run-bisect-perf-regresion.py. + run-bisect-manual-test.py + -- a script which is used to manually bisect regressions; this also + depends on bisect-perf-gression.py. + bisect-perf-regression.py + -- the main module which the others depend on. + bisect-manual-test.py + -- a helper module used when manually bisect regressions. diff --git a/tools/auto_bisect/__init__.py b/tools/auto_bisect/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/tools/auto_bisect/__init__.py diff --git a/tools/auto_bisect/bisect_utils.py b/tools/auto_bisect/bisect_utils.py new file mode 100644 index 0000000..7dc9a9b --- /dev/null +++ b/tools/auto_bisect/bisect_utils.py @@ -0,0 +1,510 @@ +# Copyright 2014 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Set of operations/utilities related to checking out the depot, and +outputting annotations on the buildbot waterfall. These are intended to be +used by the bisection scripts.""" + +import errno +import imp +import os +import shutil +import stat +import subprocess +import sys + +DEFAULT_GCLIENT_CUSTOM_DEPS = { + "src/data/page_cycler": "https://chrome-internal.googlesource.com/" + "chrome/data/page_cycler/.git", + "src/data/dom_perf": "https://chrome-internal.googlesource.com/" + "chrome/data/dom_perf/.git", + "src/data/mach_ports": "https://chrome-internal.googlesource.com/" + "chrome/data/mach_ports/.git", + "src/tools/perf/data": "https://chrome-internal.googlesource.com/" + "chrome/tools/perf/data/.git", + "src/third_party/adobe/flash/binaries/ppapi/linux": + "https://chrome-internal.googlesource.com/" + "chrome/deps/adobe/flash/binaries/ppapi/linux/.git", + "src/third_party/adobe/flash/binaries/ppapi/linux_x64": + "https://chrome-internal.googlesource.com/" + "chrome/deps/adobe/flash/binaries/ppapi/linux_x64/.git", + "src/third_party/adobe/flash/binaries/ppapi/mac": + "https://chrome-internal.googlesource.com/" + "chrome/deps/adobe/flash/binaries/ppapi/mac/.git", + "src/third_party/adobe/flash/binaries/ppapi/mac_64": + "https://chrome-internal.googlesource.com/" + "chrome/deps/adobe/flash/binaries/ppapi/mac_64/.git", + "src/third_party/adobe/flash/binaries/ppapi/win": + "https://chrome-internal.googlesource.com/" + "chrome/deps/adobe/flash/binaries/ppapi/win/.git", + "src/third_party/adobe/flash/binaries/ppapi/win_x64": + "https://chrome-internal.googlesource.com/" + "chrome/deps/adobe/flash/binaries/ppapi/win_x64/.git", + "src/chrome/tools/test/reference_build/chrome_win": None, + "src/chrome/tools/test/reference_build/chrome_mac": None, + "src/chrome/tools/test/reference_build/chrome_linux": None, + "src/third_party/WebKit/LayoutTests": None, + "src/tools/valgrind": None,} + +GCLIENT_SPEC_DATA = [ + { "name" : "src", + "url" : "https://chromium.googlesource.com/chromium/src.git", + "deps_file" : ".DEPS.git", + "managed" : True, + "custom_deps" : {}, + "safesync_url": "", + }, +] +GCLIENT_SPEC_ANDROID = "\ntarget_os = ['android']" +GCLIENT_CUSTOM_DEPS_V8 = {"src/v8_bleeding_edge": "git://github.com/v8/v8.git"} +FILE_DEPS_GIT = '.DEPS.git' +FILE_DEPS = 'DEPS' + +REPO_PARAMS = [ + 'https://chrome-internal.googlesource.com/chromeos/manifest-internal/', + '--repo-url', + 'https://git.chromium.org/external/repo.git' +] + +REPO_SYNC_COMMAND = 'git checkout -f $(git rev-list --max-count=1 '\ + '--before=%d remotes/m/master)' + +ORIGINAL_ENV = {} + +def OutputAnnotationStepStart(name): + """Outputs appropriate annotation to signal the start of a step to + a trybot. + + Args: + name: The name of the step. + """ + print + print '@@@SEED_STEP %s@@@' % name + print '@@@STEP_CURSOR %s@@@' % name + print '@@@STEP_STARTED@@@' + print + sys.stdout.flush() + + +def OutputAnnotationStepClosed(): + """Outputs appropriate annotation to signal the closing of a step to + a trybot.""" + print + print '@@@STEP_CLOSED@@@' + print + sys.stdout.flush() + + +def OutputAnnotationStepLink(label, url): + """Outputs appropriate annotation to print a link. + + Args: + label: The name to print. + url: The url to print. + """ + print + print '@@@STEP_LINK@%s@%s@@@' % (label, url) + print + sys.stdout.flush() + + +def LoadExtraSrc(path_to_file): + """Attempts to load an extra source file. If this is successful, uses the + new module to override some global values, such as gclient spec data. + + Returns: + The loaded src module, or None.""" + try: + global GCLIENT_SPEC_DATA + global GCLIENT_SPEC_ANDROID + extra_src = imp.load_source('data', path_to_file) + GCLIENT_SPEC_DATA = extra_src.GetGClientSpec() + GCLIENT_SPEC_ANDROID = extra_src.GetGClientSpecExtraParams() + return extra_src + except ImportError, e: + return None + + +def IsTelemetryCommand(command): + """Attempts to discern whether or not a given command is running telemetry.""" + return ('tools/perf/run_' in command or 'tools\\perf\\run_' in command) + + +def CreateAndChangeToSourceDirectory(working_directory): + """Creates a directory 'bisect' as a subdirectory of 'working_directory'. If + the function is successful, the current working directory will change to that + of the new 'bisect' directory. + + Returns: + True if the directory was successfully created (or already existed). + """ + cwd = os.getcwd() + os.chdir(working_directory) + try: + os.mkdir('bisect') + except OSError, e: + if e.errno != errno.EEXIST: + return False + os.chdir('bisect') + return True + + +def SubprocessCall(cmd, cwd=None): + """Runs a subprocess with specified parameters. + + Args: + params: A list of parameters to pass to gclient. + cwd: Working directory to run from. + + Returns: + The return code of the call. + """ + if os.name == 'nt': + # "HOME" isn't normally defined on windows, but is needed + # for git to find the user's .netrc file. + if not os.getenv('HOME'): + os.environ['HOME'] = os.environ['USERPROFILE'] + shell = os.name == 'nt' + return subprocess.call(cmd, shell=shell, cwd=cwd) + + +def RunGClient(params, cwd=None): + """Runs gclient with the specified parameters. + + Args: + params: A list of parameters to pass to gclient. + cwd: Working directory to run from. + + Returns: + The return code of the call. + """ + cmd = ['gclient'] + params + + return SubprocessCall(cmd, cwd=cwd) + + +def RunRepo(params): + """Runs cros repo command with specified parameters. + + Args: + params: A list of parameters to pass to gclient. + + Returns: + The return code of the call. + """ + cmd = ['repo'] + params + + return SubprocessCall(cmd) + + +def RunRepoSyncAtTimestamp(timestamp): + """Syncs all git depots to the timestamp specified using repo forall. + + Args: + params: Unix timestamp to sync to. + + Returns: + The return code of the call. + """ + repo_sync = REPO_SYNC_COMMAND % timestamp + cmd = ['forall', '-c', REPO_SYNC_COMMAND % timestamp] + return RunRepo(cmd) + + +def RunGClientAndCreateConfig(opts, custom_deps=None, cwd=None): + """Runs gclient and creates a config containing both src and src-internal. + + Args: + opts: The options parsed from the command line through parse_args(). + custom_deps: A dictionary of additional dependencies to add to .gclient. + cwd: Working directory to run from. + + Returns: + The return code of the call. + """ + spec = GCLIENT_SPEC_DATA + + if custom_deps: + for k, v in custom_deps.iteritems(): + spec[0]['custom_deps'][k] = v + + # Cannot have newlines in string on windows + spec = 'solutions =' + str(spec) + spec = ''.join([l for l in spec.splitlines()]) + + if 'android' in opts.target_platform: + spec += GCLIENT_SPEC_ANDROID + + return_code = RunGClient( + ['config', '--spec=%s' % spec, '--git-deps'], cwd=cwd) + return return_code + + +def IsDepsFileBlink(): + """Reads .DEPS.git and returns whether or not we're using blink. + + Returns: + True if blink, false if webkit. + """ + locals = {'Var': lambda _: locals["vars"][_], + 'From': lambda *args: None} + execfile(FILE_DEPS_GIT, {}, locals) + return 'blink.git' in locals['vars']['webkit_url'] + + +def OnAccessError(func, path, exc_info): + """ + Source: http://stackoverflow.com/questions/2656322/python-shutil-rmtree-fails-on-windows-with-access-is-denied + + Error handler for ``shutil.rmtree``. + + If the error is due to an access error (read only file) + it attempts to add write permission and then retries. + + If the error is for another reason it re-raises the error. + + Args: + func: The function that raised the error. + path: The path name passed to func. + exc_info: Exception information returned by sys.exc_info(). + """ + if not os.access(path, os.W_OK): + # Is the error an access error ? + os.chmod(path, stat.S_IWUSR) + func(path) + else: + raise + + +def RemoveThirdPartyDirectory(dir_name): + """Removes third_party directory from the source. + + At some point, some of the third_parties were causing issues to changes in + the way they are synced. We remove such folder in order to avoid sync errors + while bisecting. + + Returns: + True on success, otherwise False. + """ + path_to_dir = os.path.join(os.getcwd(), 'third_party', dir_name) + try: + if os.path.exists(path_to_dir): + shutil.rmtree(path_to_dir, onerror=OnAccessError) + except OSError, e: + print 'Error #%d while running shutil.rmtree(%s): %s' % ( + e.errno, path_to_dir, str(e)) + if e.errno != errno.ENOENT: + return False + return True + + +def _CleanupPreviousGitRuns(): + """Performs necessary cleanup between runs.""" + # If a previous run of git crashed, bot was reset, etc... we + # might end up with leftover index.lock files. + for (path, dir, files) in os.walk(os.getcwd()): + for cur_file in files: + if cur_file.endswith('index.lock'): + path_to_file = os.path.join(path, cur_file) + os.remove(path_to_file) + + +def RunGClientAndSync(cwd=None): + """Runs gclient and does a normal sync. + + Args: + cwd: Working directory to run from. + + Returns: + The return code of the call. + """ + params = ['sync', '--verbose', '--nohooks', '--reset', '--force'] + return RunGClient(params, cwd=cwd) + + +def SetupGitDepot(opts, custom_deps): + """Sets up the depot for the bisection. The depot will be located in a + subdirectory called 'bisect'. + + Args: + opts: The options parsed from the command line through parse_args(). + custom_deps: A dictionary of additional dependencies to add to .gclient. + + Returns: + True if gclient successfully created the config file and did a sync, False + otherwise. + """ + name = 'Setting up Bisection Depot' + + if opts.output_buildbot_annotations: + OutputAnnotationStepStart(name) + + passed = False + + if not RunGClientAndCreateConfig(opts, custom_deps): + passed_deps_check = True + if os.path.isfile(os.path.join('src', FILE_DEPS_GIT)): + cwd = os.getcwd() + os.chdir('src') + if not IsDepsFileBlink(): + passed_deps_check = RemoveThirdPartyDirectory('Webkit') + else: + passed_deps_check = True + if passed_deps_check: + passed_deps_check = RemoveThirdPartyDirectory('libjingle') + if passed_deps_check: + passed_deps_check = RemoveThirdPartyDirectory('skia') + os.chdir(cwd) + + if passed_deps_check: + _CleanupPreviousGitRuns() + + RunGClient(['revert']) + if not RunGClientAndSync(): + passed = True + + if opts.output_buildbot_annotations: + print + OutputAnnotationStepClosed() + + return passed + + +def SetupCrosRepo(): + """Sets up cros repo for bisecting chromeos. + + Returns: + Returns 0 on success. + """ + cwd = os.getcwd() + try: + os.mkdir('cros') + except OSError, e: + if e.errno != errno.EEXIST: + return False + os.chdir('cros') + + cmd = ['init', '-u'] + REPO_PARAMS + + passed = False + + if not RunRepo(cmd): + if not RunRepo(['sync']): + passed = True + os.chdir(cwd) + + return passed + + +def CopyAndSaveOriginalEnvironmentVars(): + """Makes a copy of the current environment variables.""" + # TODO: Waiting on crbug.com/255689, will remove this after. + vars_to_remove = [] + for k, v in os.environ.iteritems(): + if 'ANDROID' in k: + vars_to_remove.append(k) + vars_to_remove.append('CHROME_SRC') + vars_to_remove.append('CHROMIUM_GYP_FILE') + vars_to_remove.append('GYP_CROSSCOMPILE') + vars_to_remove.append('GYP_DEFINES') + vars_to_remove.append('GYP_GENERATORS') + vars_to_remove.append('GYP_GENERATOR_FLAGS') + vars_to_remove.append('OBJCOPY') + for k in vars_to_remove: + if os.environ.has_key(k): + del os.environ[k] + + global ORIGINAL_ENV + ORIGINAL_ENV = os.environ.copy() + + +def SetupAndroidBuildEnvironment(opts, path_to_src=None): + """Sets up the android build environment. + + Args: + opts: The options parsed from the command line through parse_args(). + path_to_src: Path to the src checkout. + + Returns: + True if successful. + """ + + # Revert the environment variables back to default before setting them up + # with envsetup.sh. + env_vars = os.environ.copy() + for k, _ in env_vars.iteritems(): + del os.environ[k] + for k, v in ORIGINAL_ENV.iteritems(): + os.environ[k] = v + + path_to_file = os.path.join('build', 'android', 'envsetup.sh') + proc = subprocess.Popen(['bash', '-c', 'source %s && env' % path_to_file], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + cwd=path_to_src) + (out, _) = proc.communicate() + + for line in out.splitlines(): + (k, _, v) = line.partition('=') + os.environ[k] = v + # envsetup.sh no longer sets OS=android to GYP_DEFINES env variable + # (CL/170273005). Set this variable explicitly inorder to build chrome on + # android. + try: + if 'OS=android' not in os.environ['GYP_DEFINES']: + os.environ['GYP_DEFINES'] = '%s %s' % (os.environ['GYP_DEFINES'], + 'OS=android') + except KeyError: + os.environ['GYP_DEFINES'] = 'OS=android' + + if opts.use_goma: + os.environ['GYP_DEFINES'] = '%s %s' % (os.environ['GYP_DEFINES'], + 'use_goma=1') + return not proc.returncode + + +def SetupPlatformBuildEnvironment(opts): + """Performs any platform specific setup. + + Args: + opts: The options parsed from the command line through parse_args(). + + Returns: + True if successful. + """ + if 'android' in opts.target_platform: + CopyAndSaveOriginalEnvironmentVars() + return SetupAndroidBuildEnvironment(opts) + elif opts.target_platform == 'cros': + return SetupCrosRepo() + + return True + + +def CheckIfBisectDepotExists(opts): + """Checks if the bisect directory already exists. + + Args: + opts: The options parsed from the command line through parse_args(). + + Returns: + Returns True if it exists. + """ + path_to_dir = os.path.join(opts.working_directory, 'bisect', 'src') + return os.path.exists(path_to_dir) + + +def CreateBisectDirectoryAndSetupDepot(opts, custom_deps): + """Sets up a subdirectory 'bisect' and then retrieves a copy of the depot + there using gclient. + + Args: + opts: The options parsed from the command line through parse_args(). + custom_deps: A dictionary of additional dependencies to add to .gclient. + """ + if not CreateAndChangeToSourceDirectory(opts.working_directory): + raise RuntimeError('Could not create bisect directory.') + + if not SetupGitDepot(opts, custom_deps): + raise RuntimeError('Failed to grab source.') diff --git a/tools/auto_bisect/post_perf_builder_job.py b/tools/auto_bisect/post_perf_builder_job.py new file mode 100644 index 0000000..97c6ecc --- /dev/null +++ b/tools/auto_bisect/post_perf_builder_job.py @@ -0,0 +1,372 @@ +# Copyright 2014 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Post a try job request via HTTP to the Tryserver to produce build.""" + +import getpass +import json +import optparse +import os +import sys +import urllib +import urllib2 + +# Link to get JSON data of builds +BUILDER_JSON_URL = ('%(server_url)s/json/builders/%(bot_name)s/builds/' + '%(build_num)s?as_text=1&filter=0') + +# Link to display build steps +BUILDER_HTML_URL = ('%(server_url)s/builders/%(bot_name)s/builds/%(build_num)s') + +# Tryserver buildbots status page +TRY_SERVER_URL = 'http://build.chromium.org/p/tryserver.chromium.perf' + +# Hostname of the tryserver where perf bisect builders are hosted. This is used +# for posting build request to tryserver. +BISECT_BUILDER_HOST = 'master4.golo.chromium.org' +# 'try_job_port' on tryserver to post build request. +BISECT_BUILDER_PORT = 8341 + + +# From buildbot.status.builder. +# See: http://docs.buildbot.net/current/developer/results.html +SUCCESS, WARNINGS, FAILURE, SKIPPED, EXCEPTION, RETRY, TRYPENDING = range(7) + +# Status codes that can be returned by the GetBuildStatus method. +OK = (SUCCESS, WARNINGS) +# Indicates build failure. +FAILED = (FAILURE, EXCEPTION, SKIPPED) +# Inidcates build in progress or in pending queue. +PENDING = (RETRY, TRYPENDING) + + +class ServerAccessError(Exception): + + def __str__(self): + return '%s\nSorry, cannot connect to server.' % self.args[0] + + +def PostTryJob(url_params): + """Sends a build request to the server using the HTTP protocol. + + Args: + url_params: A dictionary of query parameters to be sent in the request. + In order to post build request to try server, this dictionary + should contain information for following keys: + 'host': Hostname of the try server. + 'port': Port of the try server. + 'revision': SVN Revision to build. + 'bot': Name of builder bot which would be used. + Returns: + True if the request is posted successfully. Otherwise throws an exception. + """ + # Parse url parameters to be sent to Try server. + if not url_params.get('host'): + raise ValueError('Hostname of server to connect is missing.') + if not url_params.get('port'): + raise ValueError('Port of server to connect is missing.') + if not url_params.get('revision'): + raise ValueError('Missing revision details. Please specify revision' + ' information.') + if not url_params.get('bot'): + raise ValueError('Missing bot details. Please specify bot information.') + + # Pop 'host' and 'port' to avoid passing them as query params. + url = 'http://%s:%s/send_try_patch' % (url_params.pop('host'), + url_params.pop('port')) + + print 'Sending by HTTP' + query_params = '&'.join('%s=%s' % (k, v) for k, v in url_params.iteritems()) + print 'url: %s?%s' % (url, query_params) + + connection = None + try: + print 'Opening connection...' + connection = urllib2.urlopen(url, urllib.urlencode(url_params)) + print 'Done, request sent to server to produce build.' + except IOError, e: + raise ServerAccessError('%s is unaccessible. Reason: %s' % (url, e)) + if not connection: + raise ServerAccessError('%s is unaccessible.' % url) + response = connection.read() + print 'Received %s from server' % response + if response != 'OK': + raise ServerAccessError('%s is unaccessible. Got:\n%s' % (url, response)) + return True + + +def _IsBuildRunning(build_data): + """Checks whether the build is in progress on buildbot. + + Presence of currentStep element in build JSON indicates build is in progress. + + Args: + build_data: A dictionary with build data, loaded from buildbot JSON API. + + Returns: + True if build is in progress, otherwise False. + """ + current_step = build_data.get('currentStep') + if (current_step and current_step.get('isStarted') and + current_step.get('results') is None): + return True + return False + + +def _IsBuildFailed(build_data): + """Checks whether the build failed on buildbot. + + Sometime build status is marked as failed even though compile and packaging + steps are successful. This may happen due to some intermediate steps of less + importance such as gclient revert, generate_telemetry_profile are failed. + Therefore we do an addition check to confirm if build was successful by + calling _IsBuildSuccessful. + + Args: + build_data: A dictionary with build data, loaded from buildbot JSON API. + + Returns: + True if revision is failed build, otherwise False. + """ + if (build_data.get('results') in FAILED and + not _IsBuildSuccessful(build_data)): + return True + return False + + +def _IsBuildSuccessful(build_data): + """Checks whether the build succeeded on buildbot. + + We treat build as successful if the package_build step is completed without + any error i.e., when results attribute of the this step has value 0 or 1 + in its first element. + + Args: + build_data: A dictionary with build data, loaded from buildbot JSON API. + + Returns: + True if revision is successfully build, otherwise False. + """ + if build_data.get('steps'): + for item in build_data.get('steps'): + # The 'results' attribute of each step consists of two elements, + # results[0]: This represents the status of build step. + # See: http://docs.buildbot.net/current/developer/results.html + # results[1]: List of items, contains text if step fails, otherwise empty. + if (item.get('name') == 'package_build' and + item.get('isFinished') and + item.get('results')[0] in OK): + return True + return False + + +def _FetchBuilderData(builder_url): + """Fetches JSON data for the all the builds from the tryserver. + + Args: + builder_url: A tryserver URL to fetch builds information. + + Returns: + A dictionary with information of all build on the tryserver. + """ + data = None + try: + url = urllib2.urlopen(builder_url) + except urllib2.URLError, e: + print ('urllib2.urlopen error %s, waterfall status page down.[%s]' % ( + builder_url, str(e))) + return None + if url is not None: + try: + data = url.read() + except IOError, e: + print 'urllib2 file object read error %s, [%s].' % (builder_url, str(e)) + return data + + +def _GetBuildData(buildbot_url): + """Gets build information for the given build id from the tryserver. + + Args: + buildbot_url: A tryserver URL to fetch build information. + + Returns: + A dictionary with build information if build exists, otherwise None. + """ + builds_json = _FetchBuilderData(buildbot_url) + if builds_json: + return json.loads(builds_json) + return None + + +def _GetBuildBotUrl(builder_host, builder_port): + """Gets build bot URL based on the host and port of the builders. + + Note: All bisect builder bots are hosted on tryserver.chromium i.e., + on master4:8328, since we cannot access tryserver using host and port + number directly, we use tryserver URL. + + Args: + builder_host: Hostname of the server where the builder is hosted. + builder_port: Port number of ther server where the builder is hosted. + + Returns: + URL of the buildbot as a string. + """ + if (builder_host == BISECT_BUILDER_HOST and + builder_port == BISECT_BUILDER_PORT): + return TRY_SERVER_URL + else: + return 'http://%s:%s' % (builder_host, builder_port) + + +def GetBuildStatus(build_num, bot_name, builder_host, builder_port): + """Gets build status from the buildbot status page for a given build number. + + Args: + build_num: A build number on tryserver to determine its status. + bot_name: Name of the bot where the build information is scanned. + builder_host: Hostname of the server where the builder is hosted. + builder_port: Port number of ther server where the builder is hosted. + + Returns: + A tuple consists of build status (SUCCESS, FAILED or PENDING) and a link + to build status page on the waterfall. + """ + results_url = None + if build_num: + # Gets the buildbot url for the given host and port. + server_url = _GetBuildBotUrl(builder_host, builder_port) + buildbot_url = BUILDER_JSON_URL % {'server_url': server_url, + 'bot_name': bot_name, + 'build_num': build_num + } + build_data = _GetBuildData(buildbot_url) + if build_data: + # Link to build on the buildbot showing status of build steps. + results_url = BUILDER_HTML_URL % {'server_url': server_url, + 'bot_name': bot_name, + 'build_num': build_num + } + if _IsBuildFailed(build_data): + return (FAILED, results_url) + + elif _IsBuildSuccessful(build_data): + return (OK, results_url) + return (PENDING, results_url) + + +def GetBuildNumFromBuilder(build_reason, bot_name, builder_host, builder_port): + """Gets build number on build status page for a given build reason. + + It parses the JSON data from buildbot page and collect basic information + about the all the builds and then this uniquely identifies the build based + on the 'reason' attribute in builds's JSON data. + The 'reason' attribute set while a build request is posted, and same is used + to identify the build on status page. + + Args: + build_reason: A unique build name set to build on tryserver. + bot_name: Name of the bot where the build information is scanned. + builder_host: Hostname of the server where the builder is hosted. + builder_port: Port number of ther server where the builder is hosted. + + Returns: + A build number as a string if found, otherwise None. + """ + # Gets the buildbot url for the given host and port. + server_url = _GetBuildBotUrl(builder_host, builder_port) + buildbot_url = BUILDER_JSON_URL % {'server_url': server_url, + 'bot_name': bot_name, + 'build_num': '_all' + } + builds_json = _FetchBuilderData(buildbot_url) + if builds_json: + builds_data = json.loads(builds_json) + for current_build in builds_data: + if builds_data[current_build].get('reason') == build_reason: + return builds_data[current_build].get('number') + return None + + +def _GetQueryParams(options): + """Parses common query parameters which will be passed to PostTryJob. + + Args: + options: The options object parsed from the command line. + + Returns: + A dictionary consists of query parameters. + """ + values = {'host': options.host, + 'port': options.port, + 'user': options.user, + 'name': options.name + } + if options.email: + values['email'] = options.email + if options.revision: + values['revision'] = options.revision + if options.root: + values['root'] = options.root + if options.bot: + values['bot'] = options.bot + if options.patch: + values['patch'] = options.patch + return values + + +def _GenParser(): + """Parses the command line for posting build request.""" + usage = ('%prog [options]\n' + 'Post a build request to the try server for the given revision.\n') + parser = optparse.OptionParser(usage=usage) + parser.add_option('-H', '--host', + help='Host address of the try server.') + parser.add_option('-P', '--port', type='int', + help='HTTP port of the try server.') + parser.add_option('-u', '--user', default=getpass.getuser(), + dest='user', + help='Owner user name [default: %default]') + parser.add_option('-e', '--email', + default=os.environ.get('TRYBOT_RESULTS_EMAIL_ADDRESS', + os.environ.get('EMAIL_ADDRESS')), + help=('Email address where to send the results. Use either ' + 'the TRYBOT_RESULTS_EMAIL_ADDRESS environment ' + 'variable or EMAIL_ADDRESS to set the email address ' + 'the try bots report results to [default: %default]')) + parser.add_option('-n', '--name', + default='try_job_http', + help='Descriptive name of the try job') + parser.add_option('-b', '--bot', + help=('IMPORTANT: specify ONE builder per run is supported.' + 'Run script for each builders separately.')) + parser.add_option('-r', '--revision', + help=('Revision to use for the try job; default: the ' + 'revision will be determined by the try server; see ' + 'its waterfall for more info')) + parser.add_option('--root', + help=('Root to use for the patch; base subdirectory for ' + 'patch created in a subdirectory')) + parser.add_option('--patch', + help='Patch information.') + return parser + + +def Main(argv): + parser = _GenParser() + options, _ = parser.parse_args() + if not options.host: + raise ServerAccessError('Please use the --host option to specify the try ' + 'server host to connect to.') + if not options.port: + raise ServerAccessError('Please use the --port option to specify the try ' + 'server port to connect to.') + params = _GetQueryParams(options) + PostTryJob(params) + + +if __name__ == '__main__': + sys.exit(Main(sys.argv)) + |