1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
|
# Copyright 2014 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""This module contains functionality for starting build try jobs via HTTP.
This includes both sending a request to start a job, and also related code
for querying the status of the job.
This module can be either run as a stand-alone script to send a request to a
builder, or imported and used by calling the public functions below.
"""
import getpass
import json
import optparse
import os
import sys
import urllib
import urllib2
import fetch_build
# URL template for fetching JSON data about builds.
BUILDER_JSON_URL = ('%(server_url)s/json/builders/%(bot_name)s/builds/'
'%(build_num)s?as_text=1&filter=0')
# URL template for displaying build steps.
BUILDER_HTML_URL = '%(server_url)s/builders/%(bot_name)s/builds/%(build_num)s'
# Try server status page URLs, used to get build status.
PERF_TRY_SERVER_URL = 'http://build.chromium.org/p/tryserver.chromium.perf'
LINUX_TRY_SERVER_URL = 'http://build.chromium.org/p/tryserver.chromium.linux'
# Status codes that can be returned by the GetBuildStatus method
# From buildbot.status.builder.
# See: http://docs.buildbot.net/current/developer/results.html
SUCCESS, WARNINGS, FAILURE, SKIPPED, EXCEPTION, RETRY, TRYPENDING = range(7)
OK = (SUCCESS, WARNINGS) # These indicate build is complete.
FAILED = (FAILURE, EXCEPTION, SKIPPED) # These indicate build failure.
PENDING = (RETRY, TRYPENDING) # These indicate in progress or in pending queue.
class ServerAccessError(Exception):
def __str__(self):
return '%s\nSorry, cannot connect to server.' % self.args[0]
def _IsBuildRunning(build_data):
"""Checks whether the build is in progress on buildbot.
Presence of currentStep element in build JSON indicates build is in progress.
Args:
build_data: A dictionary with build data, loaded from buildbot JSON API.
Returns:
True if build is in progress, otherwise False.
"""
current_step = build_data.get('currentStep')
if (current_step and current_step.get('isStarted') and
current_step.get('results') is None):
return True
return False
def _IsBuildFailed(build_data):
"""Checks whether the build failed on buildbot.
Sometime build status is marked as failed even though compile and packaging
steps are successful. This may happen due to some intermediate steps of less
importance such as gclient revert, generate_telemetry_profile are failed.
Therefore we do an addition check to confirm if build was successful by
calling _IsBuildSuccessful.
Args:
build_data: A dictionary with build data, loaded from buildbot JSON API.
Returns:
True if revision is failed build, otherwise False.
"""
if (build_data.get('results') in FAILED and
not _IsBuildSuccessful(build_data)):
return True
return False
def _IsBuildSuccessful(build_data):
"""Checks whether the build succeeded on buildbot.
We treat build as successful if the package_build step is completed without
any error i.e., when results attribute of the this step has value 0 or 1
in its first element.
Args:
build_data: A dictionary with build data, loaded from buildbot JSON API.
Returns:
True if revision is successfully build, otherwise False.
"""
if build_data.get('steps'):
for item in build_data.get('steps'):
# The 'results' attribute of each step consists of two elements,
# results[0]: This represents the status of build step.
# See: http://docs.buildbot.net/current/developer/results.html
# results[1]: List of items, contains text if step fails, otherwise empty.
if (item.get('name') == 'package_build' and
item.get('isFinished') and
item.get('results')[0] in OK):
return True
return False
def _FetchBuilderData(builder_url):
"""Fetches JSON data for the all the builds from the tryserver.
Args:
builder_url: A tryserver URL to fetch builds information.
Returns:
A dictionary with information of all build on the tryserver.
"""
data = None
try:
url = urllib2.urlopen(builder_url)
except urllib2.URLError as e:
print ('urllib2.urlopen error %s, waterfall status page down.[%s]' % (
builder_url, str(e)))
return None
if url is not None:
try:
data = url.read()
except IOError as e:
print 'urllib2 file object read error %s, [%s].' % (builder_url, str(e))
return data
def _GetBuildData(buildbot_url):
"""Gets build information for the given build id from the tryserver.
Args:
buildbot_url: A tryserver URL to fetch build information.
Returns:
A dictionary with build information if build exists, otherwise None.
"""
builds_json = _FetchBuilderData(buildbot_url)
if builds_json:
return json.loads(builds_json)
return None
def _GetBuildBotUrl(builder_type):
"""Gets build bot URL for fetching build info.
Bisect builder bots are hosted on tryserver.chromium.perf, though we cannot
access this tryserver using host and port number directly, so we use another
tryserver URL for the perf tryserver.
Args:
builder_type: Determines what type of builder is used, e.g. "perf".
Returns:
URL of the buildbot as a string.
"""
if builder_type == fetch_build.PERF_BUILDER:
return PERF_TRY_SERVER_URL
if builder_type == fetch_build.FULL_BUILDER:
return LINUX_TRY_SERVER_URL
raise NotImplementedError('Unsupported builder type "%s".' % builder_type)
def GetBuildStatus(build_num, bot_name, builder_type):
"""Gets build status from the buildbot status page for a given build number.
Args:
build_num: A build number on tryserver to determine its status.
bot_name: Name of the bot where the build information is scanned.
builder_type: Type of builder, e.g. "perf".
Returns:
A pair which consists of build status (SUCCESS, FAILED or PENDING) and a
link to build status page on the waterfall.
"""
# TODO(prasadv, qyearsley): Make this a method of BuildArchive
# (which may be renamed to BuilderTryBot or Builder).
results_url = None
if build_num:
# Get the URL for requesting JSON data with status information.
server_url = _GetBuildBotUrl(builder_type)
buildbot_url = BUILDER_JSON_URL % {
'server_url': server_url,
'bot_name': bot_name,
'build_num': build_num,
}
build_data = _GetBuildData(buildbot_url)
if build_data:
# Link to build on the buildbot showing status of build steps.
results_url = BUILDER_HTML_URL % {
'server_url': server_url,
'bot_name': bot_name,
'build_num': build_num,
}
if _IsBuildFailed(build_data):
return (FAILED, results_url)
elif _IsBuildSuccessful(build_data):
return (OK, results_url)
return (PENDING, results_url)
def GetBuildNumFromBuilder(build_reason, bot_name, builder_type):
"""Gets build number on build status page for a given 'build reason'.
This function parses the JSON data from buildbot page and collects basic
information about the all the builds, and then uniquely identifies the build
based on the 'reason' attribute in builds's JSON data.
The 'reason' attribute set is when a build request is posted, and it is used
to identify the build on status page.
Args:
build_reason: A unique build name set to build on tryserver.
bot_name: Name of the bot where the build information is scanned.
builder_type: Type of builder, e.g. "perf".
Returns:
A build number as a string if found, otherwise None.
"""
# TODO(prasadv, qyearsley): Make this a method of BuildArchive
# (which may be renamed to BuilderTryBot or Builder).
# Gets the buildbot url for the given host and port.
server_url = _GetBuildBotUrl(builder_type)
buildbot_url = BUILDER_JSON_URL % {
'server_url': server_url,
'bot_name': bot_name,
'build_num': '_all',
}
builds_json = _FetchBuilderData(buildbot_url)
if builds_json:
builds_data = json.loads(builds_json)
for current_build in builds_data:
if builds_data[current_build].get('reason') == build_reason:
return builds_data[current_build].get('number')
return None
|
