diff options
author | prasadv <prasadv@chromium.org> | 2014-10-02 15:00:15 -0700 |
---|---|---|
committer | Commit bot <commit-bot@chromium.org> | 2014-10-02 22:00:48 +0000 |
commit | 82f6f89aedce4fd05f782e8e9b0147c5cf781b08 (patch) | |
tree | 60f6989744ecf87de992b2a98df3a269abc466ad /tools/auto_bisect | |
parent | 2178ddcc8351da2e8565df84a9812c4b23dd35f9 (diff) | |
download | chromium_src-82f6f89aedce4fd05f782e8e9b0147c5cf781b08.zip chromium_src-82f6f89aedce4fd05f782e8e9b0147c5cf781b08.tar.gz chromium_src-82f6f89aedce4fd05f782e8e9b0147c5cf781b08.tar.bz2 |
Trigger build request using git try instead of try_job_http on bisect bots
Bisect builders fail to apply patches sent via try_job_http,
because of this we were not able to build revisions with DEPS patches.
Now we are using "git try" instead to trigger build request on bisect builders.
BUG=411418
NOTRY=true
Review URL: https://codereview.chromium.org/607803005
Cr-Commit-Position: refs/heads/master@{#297918}
Diffstat (limited to 'tools/auto_bisect')
-rwxr-xr-x | tools/auto_bisect/bisect_perf_regression.py | 185 | ||||
-rw-r--r-- | tools/auto_bisect/bisect_perf_regression_test.py | 157 | ||||
-rw-r--r-- | tools/auto_bisect/bisect_utils.py | 82 |
3 files changed, 320 insertions, 104 deletions
diff --git a/tools/auto_bisect/bisect_perf_regression.py b/tools/auto_bisect/bisect_perf_regression.py index 8e94c4c..ac71428 100755 --- a/tools/auto_bisect/bisect_perf_regression.py +++ b/tools/auto_bisect/bisect_perf_regression.py @@ -176,10 +176,10 @@ HIGH_CONFIDENCE = 95 # When a build requested is posted with a patch, bisect builders on try server, # once build is produced, it reads SHA value from this file and appends it # to build archive filename. -DEPS_SHA_PATCH = """diff --git src/DEPS.sha src/DEPS.sha +DEPS_SHA_PATCH = """diff --git DEPS.sha DEPS.sha new file mode 100644 --- /dev/null -+++ src/DEPS.sha ++++ DEPS.sha @@ -0,0 +1 @@ +%(deps_sha)s """ @@ -250,6 +250,20 @@ O O | Visit http://www.chromium.org/developers/core-principles for Chrome's X | policy on perf regressions. Contact chrome-perf-dashboard-team with any / \ | questions or suggestions about bisecting. THANK YOU.""" +# Git branch name used to run bisect try jobs. +BISECT_TRYJOB_BRANCH = 'bisect-tryjob' +# Git master branch name. +BISECT_MASTER_BRANCH = 'master' +# File to store 'git diff' content. +BISECT_PATCH_FILE = 'deps_patch.txt' +# SVN repo where the bisect try jobs are submitted. +SVN_REPO_URL = 'svn://svn.chromium.org/chrome-try/try-perf' + +class RunGitError(Exception): + + def __str__(self): + return '%s\nError executing git command.' % self.args[0] + def _AddAdditionalDepotInfo(depot_info): """Adds additional depot info to the global depot variables.""" @@ -846,6 +860,95 @@ class DepotDirectoryRegistry(object): """ os.chdir(self.GetDepotDir(depot_name)) +def _PrepareBisectBranch(parent_branch, new_branch): + """Creates a new branch to submit bisect try job. + + Args: + parent_branch: Parent branch to be used to create new branch. + new_branch: New branch name. + """ + current_branch, returncode = bisect_utils.RunGit( + ['rev-parse', '--abbrev-ref', 'HEAD']) + if returncode: + raise RunGitError('Must be in a git repository to send changes to trybots.') + + current_branch = current_branch.strip() + # Make sure current branch is master. + if current_branch != parent_branch: + output, returncode = bisect_utils.RunGit(['checkout', '-f', parent_branch]) + if returncode: + raise RunGitError('Failed to checkout branch: %s.' % output) + + # Delete new branch if exists. + output, returncode = bisect_utils.RunGit(['branch', '--list' ]) + if new_branch in output: + output, returncode = bisect_utils.RunGit(['branch', '-D', new_branch]) + if returncode: + raise RunGitError('Deleting branch failed, %s', output) + + # Check if the tree is dirty: make sure the index is up to date and then + # run diff-index. + bisect_utils.RunGit(['update-index', '--refresh', '-q']) + output, returncode = bisect_utils.RunGit(['diff-index', 'HEAD']) + if output: + raise RunGitError('Cannot send a try job with a dirty tree.') + + # Create/check out the telemetry-tryjob branch, and edit the configs + # for the tryjob there. + output, returncode = bisect_utils.RunGit(['checkout', '-b', new_branch]) + if returncode: + raise RunGitError('Failed to checkout branch: %s.' % output) + + output, returncode = bisect_utils.RunGit( + ['branch', '--set-upstream-to', parent_branch]) + if returncode: + raise RunGitError('Error in git branch --set-upstream-to') + + +def _BuilderTryjob(git_revision, bot_name, bisect_job_name, patch=None): + """Attempts to run a tryjob from the current directory. + + Args: + git_revision: A Git hash revision. + bot_name: Name of the bisect bot to be used for try job. + bisect_job_name: Bisect try job name. + patch: A DEPS patch (used while bisecting 3rd party repositories). + """ + try: + # Temporary branch for running tryjob. + _PrepareBisectBranch(BISECT_MASTER_BRANCH, BISECT_TRYJOB_BRANCH) + patch_content = '/dev/null' + # Create a temporary patch file, if it fails raise an exception. + if patch: + WriteStringToFile(patch, BISECT_PATCH_FILE) + patch_content = BISECT_PATCH_FILE + + try_cmd = ['try', + '-b', bot_name, + '-r', git_revision, + '-n', bisect_job_name, + '--svn_repo=%s' % SVN_REPO_URL, + '--diff=%s' % patch_content + ] + # Execute try job to build revision. + output, returncode = bisect_utils.RunGit(try_cmd) + + if returncode: + raise RunGitError('Could not execute tryjob: %s.\n Error: %s' % ( + 'git %s' % ' '.join(try_cmd), output)) + print ('Try job successfully submitted.\n TryJob Details: %s\n%s' % ( + 'git %s' % ' '.join(try_cmd), output)) + finally: + # Delete patch file if exists + try: + os.remove(BISECT_PATCH_FILE) + except OSError as e: + if e.errno != errno.ENOENT: + raise + # Checkout master branch and delete bisect-tryjob branch. + bisect_utils.RunGit(['checkout', '-f', BISECT_MASTER_BRANCH]) + bisect_utils.RunGit(['branch', '-D', BISECT_TRYJOB_BRANCH]) + class BisectPerformanceMetrics(object): """This class contains functionality to perform a bisection of a range of @@ -1182,7 +1285,7 @@ class BisectPerformanceMetrics(object): return FetchFromCloudStorage(gs_bucket, source_file, out_dir) return downloaded_archive - def DownloadCurrentBuild(self, revision, build_type='Release', patch=None): + def DownloadCurrentBuild(self, revision, depot, build_type='Release'): """Downloads the build archive for the given revision. Args: @@ -1193,7 +1296,12 @@ class BisectPerformanceMetrics(object): Returns: True if download succeeds, otherwise False. """ + patch = None patch_sha = None + if depot != 'chromium': + # Create a DEPS patch with new revision for dependency repository. + revision, patch = self.CreateDEPSPatch(depot, revision) + if patch: # Get the SHA of the DEPS changes patch. patch_sha = GetSHA1HexDigest(patch) @@ -1290,37 +1398,31 @@ class BisectPerformanceMetrics(object): if not fetch_build: return False - bot_name, build_timeout = GetBuilderNameAndBuildTime( - self.opts.target_platform, self.opts.target_arch) - builder_host = self.opts.builder_host - builder_port = self.opts.builder_port # Create a unique ID for each build request posted to try server builders. # This ID is added to "Reason" property of the build. build_request_id = GetSHA1HexDigest( '%s-%s-%s' % (git_revision, patch, time.time())) - # Creates a try job description. - # Always use Git hash to post build request since Commit positions are - # not supported by builders to build. - job_args = { - 'revision': 'src@%s' % git_revision, - 'bot': bot_name, - 'name': build_request_id, - } - # Update patch information if supplied. - if patch: - job_args['patch'] = patch - # Posts job to build the revision on the server. - if request_build.PostTryJob(builder_host, builder_port, job_args): + # Reverts any changes to DEPS file. + self.source_control.CheckoutFileAtRevision( + bisect_utils.FILE_DEPS, git_revision, cwd=self.src_cwd) + + bot_name, build_timeout = GetBuilderNameAndBuildTime( + self.opts.target_platform, self.opts.target_arch) + target_file = None + try: + # Execute try job request to build revision with patch. + _BuilderTryjob(git_revision, bot_name, build_request_id, patch) target_file, error_msg = _WaitUntilBuildIsReady( - fetch_build, bot_name, builder_host, builder_port, build_request_id, - build_timeout) + fetch_build, bot_name, self.opts.builder_host, + self.opts.builder_port, build_request_id, build_timeout) if not target_file: print '%s [revision: %s]' % (error_msg, git_revision) - return None - return target_file - print 'Failed to post build request for revision: [%s]' % git_revision - return None + except RunGitError as e: + print ('Failed to post builder try job for revision: [%s].\n' + 'Error: %s' % (git_revision, e)) + + return target_file def IsDownloadable(self, depot): """Checks if build can be downloaded based on target platform and depot.""" @@ -1417,6 +1519,7 @@ class BisectPerformanceMetrics(object): print 'Something went wrong while updating DEPS file. [%s]' % e return False + def CreateDEPSPatch(self, depot, revision): """Modifies DEPS and returns diff as text. @@ -1444,8 +1547,8 @@ class BisectPerformanceMetrics(object): if self.UpdateDeps(revision, depot, deps_file_path): diff_command = [ 'diff', - '--src-prefix=src/', - '--dst-prefix=src/', + '--src-prefix=', + '--dst-prefix=', '--no-ext-diff', bisect_utils.FILE_DEPS, ] @@ -1474,16 +1577,7 @@ class BisectPerformanceMetrics(object): # Fetch build archive for the given revision from the cloud storage when # the storage bucket is passed. if self.IsDownloadable(depot) and revision: - deps_patch = None - if depot != 'chromium': - # Create a DEPS patch with new revision for dependency repository. - revision, deps_patch = self.CreateDEPSPatch(depot, revision) - if self.DownloadCurrentBuild(revision, patch=deps_patch): - if deps_patch: - # Reverts the changes to DEPS file. - self.source_control.CheckoutFileAtRevision( - bisect_utils.FILE_DEPS, revision, cwd=self.src_cwd) - build_success = True + build_success = self.DownloadCurrentBuild(revision, depot) else: # These codes are executed when bisect bots builds binaries locally. build_success = self.builder.Build(depot, self.opts) @@ -1779,17 +1873,7 @@ class BisectPerformanceMetrics(object): Returns: True if successful. """ - if depot == 'chromium' or depot == 'android-chrome': - # Removes third_party/libjingle. At some point, libjingle was causing - # issues syncing when using the git workflow (crbug.com/266324). - os.chdir(self.src_cwd) - if not bisect_utils.RemoveThirdPartyDirectory('libjingle'): - return False - # Removes third_party/skia. At some point, skia was causing - # issues syncing when using the git workflow (crbug.com/377951). - if not bisect_utils.RemoveThirdPartyDirectory('skia'): - return False - elif depot == 'cros': + if depot == 'cros': return self.PerformCrosChrootCleanup() return True @@ -2639,7 +2723,8 @@ class BisectPerformanceMetrics(object): current_data['depot']) if not cl_link: cl_link = current_id - commit_position = self.source_control.GetCommitPosition(current_id) + commit_position = self.source_control.GetCommitPosition( + current_id, self.depot_registry.GetDepotDir(current_data['depot'])) commit_position = str(commit_position) if not commit_position: commit_position = '' diff --git a/tools/auto_bisect/bisect_perf_regression_test.py b/tools/auto_bisect/bisect_perf_regression_test.py index b8f32266..0e56e44 100644 --- a/tools/auto_bisect/bisect_perf_regression_test.py +++ b/tools/auto_bisect/bisect_perf_regression_test.py @@ -5,12 +5,18 @@ import os import re import shutil +import sys import unittest +SRC = os.path.join(os.path.dirname(__file__), os.path.pardir, os.path.pardir) +sys.path.append(os.path.join(SRC, 'third_party', 'pymock')) + import bisect_perf_regression import bisect_results +import mock import source_control as source_control_module + def _GetBisectPerformanceMetricsInstance(): """Returns an instance of the BisectPerformanceMetrics class.""" options_dict = { @@ -348,5 +354,156 @@ class DepotDirectoryRegistryTest(unittest.TestCase): self.assertEqual(self.cur_dir, '/mock/src/foo') +# The tests below test private functions (W0212). +# pylint: disable=W0212 +class GitTryJobTestCases(unittest.TestCase): + """Test case for bisect try job.""" + def setUp(self): + bisect_utils_patcher = mock.patch('bisect_perf_regression.bisect_utils') + self.mock_bisect_utils = bisect_utils_patcher.start() + self.addCleanup(bisect_utils_patcher.stop) + + def _SetupRunGitMock(self, git_cmds): + """Setup RunGit mock with expected output for given git command.""" + def side_effect(git_cmd_args): + for val in git_cmds: + if set(val[0]) == set(git_cmd_args): + return val[1] + self.mock_bisect_utils.RunGit = mock.Mock(side_effect=side_effect) + + def _AssertRunGitExceptions(self, git_cmds, func, *args): + """Setup RunGit mock and tests RunGitException. + + Args: + git_cmds: List of tuples with git command and expected output. + func: Callback function to be executed. + args: List of arguments to be passed to the function. + """ + self._SetupRunGitMock(git_cmds) + self.assertRaises(bisect_perf_regression.RunGitError, + func, + *args) + + def testNotGitRepo(self): + new_branch = bisect_perf_regression.BISECT_TRYJOB_BRANCH + parent_branch = bisect_perf_regression.BISECT_MASTER_BRANCH + cmds = [(['rev-parse', '--abbrev-ref', 'HEAD'], (None, 128))] + self._AssertRunGitExceptions(cmds, + bisect_perf_regression._PrepareBisectBranch, + parent_branch, new_branch) + + def testFailedCheckoutMaster(self): + new_branch = bisect_perf_regression.BISECT_TRYJOB_BRANCH + parent_branch = bisect_perf_regression.BISECT_MASTER_BRANCH + cmds = [ + (['rev-parse', '--abbrev-ref', 'HEAD'], (new_branch, 0)), + (['checkout', '-f', parent_branch], ('Checkout Failed', 1))] + self._AssertRunGitExceptions(cmds, + bisect_perf_regression._PrepareBisectBranch, + parent_branch, new_branch) + + def testDeleteBisectBranchIfExists(self): + new_branch = bisect_perf_regression.BISECT_TRYJOB_BRANCH + parent_branch = bisect_perf_regression.BISECT_MASTER_BRANCH + cmds = [ + (['rev-parse', '--abbrev-ref', 'HEAD'], (parent_branch, 0)), + (['branch', '--list' ], ('bisect-tryjob\n*master\nsomebranch', 0)), + (['branch', '-D', new_branch], ('Failed to delete branch', 128)) + ] + self._AssertRunGitExceptions(cmds, + bisect_perf_regression._PrepareBisectBranch, + parent_branch, new_branch) + + def testCreatNewBranchFails(self): + new_branch = bisect_perf_regression.BISECT_TRYJOB_BRANCH + parent_branch = bisect_perf_regression.BISECT_MASTER_BRANCH + cmds = [ + (['rev-parse', '--abbrev-ref', 'HEAD'], (parent_branch, 0)), + (['branch', '--list' ], ('bisect-tryjob\n*master\nsomebranch', 0)), + (['branch', '-D', new_branch], ('None', 0)), + (['update-index', '--refresh', '-q'], (None, 0)), + (['diff-index', 'HEAD'], (None, 0)), + (['checkout', '-b', new_branch], ('Failed to create branch', 128)) + ] + + self._AssertRunGitExceptions(cmds, + bisect_perf_regression._PrepareBisectBranch, + parent_branch, new_branch) + + def testSetUpstreamToFails(self): + new_branch = bisect_perf_regression.BISECT_TRYJOB_BRANCH + parent_branch = bisect_perf_regression.BISECT_MASTER_BRANCH + cmds = [ + (['rev-parse', '--abbrev-ref', 'HEAD'], (parent_branch, 0)), + (['branch', '--list' ], ('bisect-tryjob\n*master\nsomebranch', 0)), + (['branch', '-D', new_branch], ('None', 0)), + (['update-index', '--refresh', '-q'], (None, 0)), + (['diff-index', 'HEAD'], (None, 0)), + (['checkout', '-b', new_branch], ('None', 0)), + (['branch', '--set-upstream-to', parent_branch], + ('Setuptream fails', 1)) + ] + self._AssertRunGitExceptions(cmds, + bisect_perf_regression._PrepareBisectBranch, + parent_branch, new_branch) + def testBuilderTryJobForException(self): + git_revision = 'ac4a9f31fe2610bd146857bbd55d7a260003a888' + bot_name = 'linux_perf_bisect_builder' + bisect_job_name = 'testBisectJobname' + patch = None + patch_content = '/dev/null' + new_branch = bisect_perf_regression.BISECT_TRYJOB_BRANCH + parent_branch = bisect_perf_regression.BISECT_MASTER_BRANCH + try_cmd = [ + (['rev-parse', '--abbrev-ref', 'HEAD'], (parent_branch, 0)), + (['branch', '--list' ], ('bisect-tryjob\n*master\nsomebranch', 0)), + (['branch', '-D', new_branch], ('None', 0)), + (['update-index', '--refresh', '-q'], (None, 0)), + (['diff-index', 'HEAD'], (None, 0)), + (['checkout', '-b', new_branch], ('None', 0)), + (['branch', '--set-upstream-to', parent_branch], + ('Setuptream fails', 0)), + (['try', + '-b', bot_name, + '-r', git_revision, + '-n', bisect_job_name, + '--svn_repo=%s' % bisect_perf_regression.SVN_REPO_URL, + '--diff=%s' % patch_content + ], (None, 1)) + ] + self._AssertRunGitExceptions(try_cmd, + bisect_perf_regression._BuilderTryjob, + git_revision, bot_name, bisect_job_name, patch) + + def testBuilderTryJob(self): + git_revision = 'ac4a9f31fe2610bd146857bbd55d7a260003a888' + bot_name = 'linux_perf_bisect_builder' + bisect_job_name = 'testBisectJobname' + patch = None + patch_content = '/dev/null' + new_branch = bisect_perf_regression.BISECT_TRYJOB_BRANCH + parent_branch = bisect_perf_regression.BISECT_MASTER_BRANCH + try_cmd = [ + (['rev-parse', '--abbrev-ref', 'HEAD'], (parent_branch, 0)), + (['branch', '--list' ], ('bisect-tryjob\n*master\nsomebranch', 0)), + (['branch', '-D', new_branch], ('None', 0)), + (['update-index', '--refresh', '-q'], (None, 0)), + (['diff-index', 'HEAD'], (None, 0)), + (['checkout', '-b', new_branch], ('None', 0)), + (['branch', '--set-upstream-to', parent_branch], + ('Setuptream fails', 0)), + (['try', + '-b', bot_name, + '-r', git_revision, + '-n', bisect_job_name, + '--svn_repo=%s' % bisect_perf_regression.SVN_REPO_URL, + '--diff=%s' % patch_content + ], (None, 0)) + ] + self._SetupRunGitMock(try_cmd) + bisect_perf_regression._BuilderTryjob( + git_revision, bot_name, bisect_job_name, patch) + + if __name__ == '__main__': unittest.main() diff --git a/tools/auto_bisect/bisect_utils.py b/tools/auto_bisect/bisect_utils.py index f75c976..9dfc652 100644 --- a/tools/auto_bisect/bisect_utils.py +++ b/tools/auto_bisect/bisect_utils.py @@ -11,7 +11,6 @@ annotations for the Buildbot waterfall. import errno import imp import os -import shutil import stat import subprocess import sys @@ -83,6 +82,9 @@ REPO_PARAMS = [ 'https://git.chromium.org/external/repo.git' ] +# Bisect working directory. +BISECT_DIR = 'bisect' + def OutputAnnotationStepStart(name): """Outputs annotation to signal the start of a step to a try bot. @@ -162,12 +164,12 @@ def _CreateAndChangeToSourceDirectory(working_directory): cwd = os.getcwd() os.chdir(working_directory) try: - os.mkdir('bisect') + os.mkdir(BISECT_DIR) except OSError, e: if e.errno != errno.EEXIST: # EEXIST indicates that it already exists. os.chdir(cwd) return False - os.chdir('bisect') + os.chdir(BISECT_DIR) return True @@ -308,28 +310,6 @@ def OnAccessError(func, path, _): raise -def RemoveThirdPartyDirectory(dir_name): - """Removes third_party directory from the source. - - At some point, some of the third_parties were causing issues to changes in - the way they are synced. We remove such folder in order to avoid sync errors - while bisecting. - - Returns: - True on success, otherwise False. - """ - path_to_dir = os.path.join(os.getcwd(), 'third_party', dir_name) - try: - if os.path.exists(path_to_dir): - shutil.rmtree(path_to_dir, onerror=OnAccessError) - except OSError, e: - print 'Error #%d while running shutil.rmtree(%s): %s' % ( - e.errno, path_to_dir, str(e)) - if e.errno != errno.ENOENT: - return False - return True - - def _CleanupPreviousGitRuns(): """Cleans up any leftover index.lock files after running git.""" # If a previous run of git crashed, or bot was reset, etc., then we might @@ -350,7 +330,8 @@ def RunGClientAndSync(cwd=None): Returns: The return code of the call. """ - params = ['sync', '--verbose', '--nohooks', '--reset', '--force'] + params = ['sync', '--verbose', '--nohooks', '--reset', '--force', + '--delete_unversioned_trees'] return RunGClient(params, cwd=cwd) @@ -368,35 +349,19 @@ def SetupGitDepot(opts, custom_deps): otherwise. """ name = 'Setting up Bisection Depot' + try: + if opts.output_buildbot_annotations: + OutputAnnotationStepStart(name) - if opts.output_buildbot_annotations: - OutputAnnotationStepStart(name) - - passed = False - - if not RunGClientAndCreateConfig(opts, custom_deps): - passed_deps_check = True - if os.path.isfile(os.path.join('src', FILE_DEPS_GIT)): - cwd = os.getcwd() - os.chdir('src') - if passed_deps_check: - passed_deps_check = RemoveThirdPartyDirectory('libjingle') - if passed_deps_check: - passed_deps_check = RemoveThirdPartyDirectory('skia') - os.chdir(cwd) - - if passed_deps_check: - _CleanupPreviousGitRuns() - - RunGClient(['revert']) - if not RunGClientAndSync(): - passed = True + if RunGClientAndCreateConfig(opts, custom_deps): + return False - if opts.output_buildbot_annotations: - print - OutputAnnotationStepClosed() - - return passed + _CleanupPreviousGitRuns() + RunGClient(['revert']) + return not RunGClientAndSync() + finally: + if opts.output_buildbot_annotations: + OutputAnnotationStepClosed() def CheckIfBisectDepotExists(opts): @@ -408,7 +373,7 @@ def CheckIfBisectDepotExists(opts): Returns: Returns True if it exists. """ - path_to_dir = os.path.join(opts.working_directory, 'bisect', 'src') + path_to_dir = os.path.join(opts.working_directory, BISECT_DIR, 'src') return os.path.exists(path_to_dir) @@ -451,6 +416,15 @@ def CreateBisectDirectoryAndSetupDepot(opts, custom_deps): opts: The options parsed from the command line through parse_args(). custom_deps: A dictionary of additional dependencies to add to .gclient. """ + if CheckIfBisectDepotExists(opts): + path_to_dir = os.path.join(os.path.abspath(opts.working_directory), + BISECT_DIR, 'src') + (output, _) = RunGit(['rev-parse', '--is-inside-work-tree'], + cwd=path_to_dir) + if output.strip() == 'true': + # Checks out the master branch, throws an exception if git command fails. + CheckRunGit(['checkout', '-f', 'master'], cwd=path_to_dir) + if not _CreateAndChangeToSourceDirectory(opts.working_directory): raise RuntimeError('Could not create bisect directory.') |