diff options
author | chrisphan <chrisphan@chromium.org> | 2016-02-22 14:37:55 -0800 |
---|---|---|
committer | Commit bot <commit-bot@chromium.org> | 2016-02-22 22:39:04 +0000 |
commit | 4550f726fecaa39bb479626907a4a80bbd702c3d (patch) | |
tree | eb66156292411708c7f317e1207849f615a655d8 /tools/auto_bisect | |
parent | fdb9a2bea13a877eef407fe7359b056b56d8ab2e (diff) | |
download | chromium_src-4550f726fecaa39bb479626907a4a80bbd702c3d.zip chromium_src-4550f726fecaa39bb479626907a4a80bbd702c3d.tar.gz chromium_src-4550f726fecaa39bb479626907a4a80bbd702c3d.tar.bz2 |
Update legacy bisect to post results to perf dashboard.
Since we're moving away from this, no change in the current results reporting here.
BUG=573308
Review URL: https://codereview.chromium.org/1625573004
Cr-Commit-Position: refs/heads/master@{#376829}
Diffstat (limited to 'tools/auto_bisect')
-rwxr-xr-x | tools/auto_bisect/bisect_perf_regression.py | 26 | ||||
-rw-r--r-- | tools/auto_bisect/bisect_perf_regression_test.py | 58 | ||||
-rw-r--r-- | tools/auto_bisect/bisect_results.py | 14 | ||||
-rw-r--r-- | tools/auto_bisect/bisect_results_json.py | 88 |
4 files changed, 178 insertions, 8 deletions
diff --git a/tools/auto_bisect/bisect_perf_regression.py b/tools/auto_bisect/bisect_perf_regression.py index fb57afc..dc8d199 100755 --- a/tools/auto_bisect/bisect_perf_regression.py +++ b/tools/auto_bisect/bisect_perf_regression.py @@ -41,11 +41,12 @@ try bots, and is started by tools/run-bisect-perf-regression.py using config parameters from tools/auto_bisect/bisect.cfg. """ +import argparse import copy import errno import hashlib +import json import logging -import argparse import os import re import shlex @@ -53,12 +54,14 @@ import shutil import StringIO import sys import time +import urllib2 sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'third_party', 'catapult', 'telemetry')) from bisect_printer import BisectPrinter from bisect_results import BisectResults +import bisect_results_json from bisect_state import BisectState import bisect_utils import builder @@ -124,6 +127,7 @@ PERF_SVN_REPO_URL = 'svn://svn.chromium.org/chrome-try/try-perf' FULL_SVN_REPO_URL = 'svn://svn.chromium.org/chrome-try/try' ANDROID_CHROME_SVN_REPO_URL = ('svn://svn.chromium.org/chrome-try-internal/' 'try-perf') +PERF_DASH_RESULTS_URL = 'https://chromeperf.appspot.com/post_bisect_results' class RunGitError(Exception): @@ -2505,6 +2509,24 @@ def _IsPlatformSupported(): return os.name in supported +def _PostBisectResults(bisect_results, opts, src_cwd): + """Posts bisect results to Perf Dashboard.""" + bisect_utils.OutputAnnotationStepStart('Post Results') + + results = bisect_results_json.Get( + bisect_results, opts, DepotDirectoryRegistry(src_cwd)) + data = {'data': results} + request = urllib2.Request(PERF_DASH_RESULTS_URL) + request.add_header('Content-Type', 'application/json') + try: + urllib2.urlopen(request, json.dumps(data)) + except urllib2.URLError as e: + print 'Failed to post bisect results. Error: %s.' % e + bisect_utils.OutputAnnotationStepWarning() + + bisect_utils.OutputAnnotationStepClosed() + + def RemoveBuildFiles(build_type): """Removes build files from previous runs.""" out_dir = os.path.join('out', build_type) @@ -2584,6 +2606,7 @@ class BisectOptions(object): self.improvement_direction = 0 self.bug_id = '' self.required_initial_confidence = 80.0 + self.try_job_id = None @staticmethod def _AddBisectOptionsGroup(parser): @@ -2858,6 +2881,7 @@ def main(): if results.error: raise RuntimeError(results.error) bisect_test.printer.FormatAndPrintResults(results) + _PostBisectResults(results, opts, os.getcwd()) return 0 finally: bisect_test.PerformCleanup() diff --git a/tools/auto_bisect/bisect_perf_regression_test.py b/tools/auto_bisect/bisect_perf_regression_test.py index 33f733a..918334b 100644 --- a/tools/auto_bisect/bisect_perf_regression_test.py +++ b/tools/auto_bisect/bisect_perf_regression_test.py @@ -12,6 +12,8 @@ SRC = os.path.join(os.path.dirname(__file__), os.path.pardir, os.path.pardir) sys.path.append(os.path.join(SRC, 'third_party', 'pymock')) import bisect_perf_regression +import bisect_results +import bisect_state import bisect_utils import fetch_build import mock @@ -131,6 +133,47 @@ def _FakeTestResult(values, bisect_mode_is_return_code): return (result_dict, success_code) +def _SampleBisecResult(opts): + revisions = [ + 'ae7ef14ba2d9b5ef0d2c1c092ec98a417e44740d' + 'ab55ead638496b061c9de61685b982f7cea38ca7', + '89aa0c99e4b977b9a4f992ac14da0d6624f7316e'] + state = bisect_state.BisectState(depot='chromium', revisions=revisions) + depot_registry = bisect_perf_regression.DepotDirectoryRegistry('/mock/src') + results = bisect_results.BisectResults( + bisect_state=state, depot_registry=depot_registry, opts=opts, + runtime_warnings=[]) + results.confidence = 99.9 + results.culprit_revisions = [( + 'ab55ead638496b061c9de61685b982f7cea38ca7', + { + 'date': 'Thu, 26 Jun 2014 14:29:49 +0000', + 'body': 'Fix', + 'author': 'author@chromium.org', + 'subject': 'Fix', + 'email': 'author@chromium.org', + }, + 'chromium')] + return results + + +def _GetMockCallArg(function_mock, call_index): + """Gets the list of called arguments for call at |call_index|. + + Args: + function_mock: A Mock object. + call_index: The index at which the mocked function was called. + + Returns: + The called argument list. + """ + call_args_list = function_mock.call_args_list + if not call_args_list or len(call_args_list) <= call_index: + return None + args, _ = call_args_list[call_index] + return args + + def _GetBisectPerformanceMetricsInstance(options_dict): """Returns an instance of the BisectPerformanceMetrics class.""" opts = bisect_perf_regression.BisectOptions.FromDict(options_dict) @@ -319,6 +362,21 @@ class BisectPerfRegressionTest(unittest.TestCase): results = _GenericDryRun(_GetExtendedOptions(1, -100)) self.assertIsNone(results.error) + @mock.patch('urllib2.urlopen') + def testBisectResultsPosted(self, mock_urlopen): + options_dict = dict(DEFAULT_OPTIONS) + options_dict.update({ + 'bisect_mode': bisect_utils.BISECT_MODE_MEAN, + 'try_job_id': 1234, + }) + opts = bisect_perf_regression.BisectOptions.FromDict(options_dict) + results = _SampleBisecResult(opts) + bisect_perf_regression._PostBisectResults(results, opts, os.getcwd()) + + call_args = _GetMockCallArg(mock_urlopen, 0) + self.assertIsNotNone(call_args) + self.assertIn('"try_job_id": 1234', call_args[1]) + def _CheckAbortsEarly(self, results, **extra_opts): """Returns True if the bisect job would abort early.""" global _MockResultsGenerator diff --git a/tools/auto_bisect/bisect_results.py b/tools/auto_bisect/bisect_results.py index f6ba0d8..094d107 100644 --- a/tools/auto_bisect/bisect_results.py +++ b/tools/auto_bisect/bisect_results.py @@ -56,6 +56,13 @@ class BisectResults(object): runtime_warnings: A list of warnings from the bisect run. error: Error message. When error is not None, other arguments are ignored. """ + # Setting these attributes so that bisect printer does not break when the + # regression cannot be reproduced (no broken revision was found) + self.regression_size = 0 + self.regression_std_err = 0 + self.confidence = 0 + self.culprit_revisions = [] + self.error = error self.abort_reason = abort_reason if error is not None or abort_reason is not None: @@ -91,13 +98,6 @@ class BisectResults(object): self.warnings += self._GetResultBasedWarnings( self.culprit_revisions, opts, self.confidence) - elif first_working_rev is not None: - # Setting these attributes so that bisect printer does not break when the - # regression cannot be reproduced (no broken revision was found) - self.regression_size = 0 - self.regression_std_err = 0 - self.confidence = 0 - self.culprit_revisions = [] def AddRetestResults(self, results_tot, results_reverted): if not results_tot or not results_reverted: diff --git a/tools/auto_bisect/bisect_results_json.py b/tools/auto_bisect/bisect_results_json.py new file mode 100644 index 0000000..210159c --- /dev/null +++ b/tools/auto_bisect/bisect_results_json.py @@ -0,0 +1,88 @@ +# Copyright 2016 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import os + +import bisect_utils +import source_control + + +def Get(bisect_results, opts, depot_registry): + """Returns the results as a jsonable object.""" + if opts.bisect_mode == bisect_utils.BISECT_MODE_RETURN_CODE: + change = '0' + else: + metric = '/'.join(opts.metric) + change = '%.02f%%' % bisect_results.regression_size + + status = 'completed' + + return { + 'try_job_id': opts.try_job_id, + 'bug_id': opts.bug_id, + 'status': status, + 'buildbot_log_url': _GetBuildBotLogUrl(), + 'bisect_bot': os.environ.get('BUILDBOT_BUILDERNAME', ''), + 'command': opts.command, + 'metric': metric, + 'change': change, + 'score': bisect_results.confidence, + 'good_revision': opts.good_revision, + 'bad_revision': opts.bad_revision, + 'warnings': bisect_results.warnings, + 'abort_reason': bisect_results.abort_reason, + 'culprit_data': _CulpritData(bisect_results), + 'revision_data': _RevisionData(bisect_results, depot_registry), + } + + +def _CulpritData(bisect_results): + if not bisect_results.culprit_revisions: + return None + cl, culprit_info, depot = bisect_results.culprit_revisions[0] + commit_link = _GetViewVCLinkFromDepotAndHash(cl, depot) + if commit_link: + commit_link = '\nLink : %s' % commit_link + else: + commit_link = ('\Description:\n%s' % culprit_info['body']) + + return { + 'subject': culprit_info['subject'], + 'author': culprit_info['email'], + 'email': culprit_info['email'], + 'cl_date': culprit_info['date'], + 'commit_info': commit_link, + 'revisions_links': [], + 'cl': cl + } + + +def _RevisionData(bisect_results, depot_registry): + revision_rows = [] + for state in bisect_results.state.GetRevisionStates(): + commit_position = source_control.GetCommitPosition( + state.revision, depot_registry.GetDepotDir(state.depot)) + revision_rows.append({ + 'depot_name': state.depot, + 'deps_revision': state.revision, + 'commit_pos': commit_position, + 'result': 'good' if state.passed else 'bad', + }) + return revision_rows + + +def _GetViewVCLinkFromDepotAndHash(git_revision, depot): + """Gets link to the repository browser.""" + if depot and 'viewvc' in bisect_utils.DEPOT_DEPS_NAME[depot]: + return bisect_utils.DEPOT_DEPS_NAME[depot]['viewvc'] + git_revision + return '' + + +def _GetBuildBotLogUrl(): + master_url = os.environ.get('BUILDBOT_BUILDBOTURL') + builder_name = os.environ.get('BUILDBOT_BUILDERNAME') + builder_number = os.environ.get('BUILDBOT_BUILDNUMBER') + if master_url and builder_name and builder_number: + return '%s%s/%s' % (master_url, builder_name, builder_number) + return '' |