1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
|
# Copyright 2014 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import os
import unittest
from bisect_results import BisectResults
import source_control
class MockDepotRegistry(object):
def ChangeToDepotDir(self, depot):
pass
class MockRevisionState(object):
def __init__(self, revision, index, depot='chromium', value=None,
perf_time=0, build_time=0, passed='?', external=None):
self.depot = depot
self.revision = revision
self.index = index
self.value = value
self.perf_time = perf_time
self.build_time = build_time
self.passed = passed
self.external = external
class MockBisectState(object):
def __init__(self):
self.mock_revision_states = []
mock_bad_val = {'values': [100, 105, 95]}
for i, rev in enumerate(['a', 'b']):
mock_rev_state = MockRevisionState(rev, i, value=mock_bad_val, passed=0)
self.mock_revision_states.append(mock_rev_state)
mock_good_val = {'values': [1, 2, 3]}
for i, rev in enumerate(['c', 'd', 'e'], start=2):
mock_rev_state = MockRevisionState(rev, i, value=mock_good_val, passed=1)
self.mock_revision_states.append(mock_rev_state)
def GetRevisionStates(self):
return self.mock_revision_states
class MockBisectOptions(object):
def __init__(self):
self.repeat_test_count = 3
class BisectResultsTest(unittest.TestCase):
def setUp(self):
self.mock_bisect_state = MockBisectState()
self.mock_depot_registry = MockDepotRegistry()
self.mock_opts = MockBisectOptions()
self.mock_warnings = []
self.original_getcwd = os.getcwd
self.original_chdir = os.chdir
self.original_query_revision_info = source_control.QueryRevisionInfo
os.getcwd = lambda: '/path'
os.chdir = lambda _: None
revision_infos = {'b': {'test': 'b'}, 'c': {'test': 'c'}}
source_control.QueryRevisionInfo = lambda rev: revision_infos[rev]
def tearDown(self):
os.getcwd = self.original_getcwd
os.chdir = self.original_chdir
source_control.QueryRevisionInfo = self.original_query_revision_info
def _AssertConfidence(self, score, bad_values, good_values):
"""Checks whether the given sets of values have a given confidence score.
The score represents our confidence that the two sets of values wouldn't
be as different as they are just by chance; that is, that some real change
occurred between the two sets of values.
Args:
score: Expected confidence score.
bad_values: First list of numbers.
good_values: Second list of numbers.
"""
# ConfidenceScore takes a list of lists but these lists are flattened
# inside the function.
confidence = BisectResults.ConfidenceScore(
[[v] for v in bad_values],
[[v] for v in good_values])
self.assertEqual(score, confidence)
def testConfidenceScoreIsZeroOnTooFewLists(self):
self._AssertConfidence(0.0, [], [[1], [2]])
self._AssertConfidence(0.0, [[1], [2]], [])
self._AssertConfidence(0.0, [[1]], [[1], [2]])
self._AssertConfidence(0.0, [[1], [2]], [[1]])
def testConfidenceScoreIsZeroOnEmptyLists(self):
self.assertEqual(BisectResults.ConfidenceScore([[], []], [[1], [2]]), 0.0)
self.assertEqual(BisectResults.ConfidenceScore([[1], [2]], [[], []]), 0.0)
def testConfidenceScore_ZeroConfidence(self):
# The good and bad sets contain the same values, so the confidence that
# they're different should be zero.
self._AssertConfidence(0.0, [4, 5, 7, 6, 8, 7], [8, 7, 6, 7, 5, 4])
def testConfidenceScore_MediumConfidence(self):
self._AssertConfidence(80.0, [0, 1, 1, 1, 2, 2], [1, 1, 1, 3, 3, 4])
def testConfidenceScore_HighConfidence(self):
self._AssertConfidence(95.0, [0, 1, 1, 1, 2, 2], [1, 2, 2, 3, 3, 4])
def testConfidenceScore_VeryHighConfidence(self):
# Confidence is high if the two sets of values have no internal variance.
self._AssertConfidence(99.9, [1, 1, 1, 1], [1.2, 1.2, 1.2, 1.2])
self._AssertConfidence(99.9, [1, 1, 1, 1], [1.01, 1.01, 1.01, 1.01])
def testConfidenceScore_UnbalancedSampleSize(self):
# The second set of numbers only contains one number, so confidence is 0.
self._AssertConfidence(0.0, [1.1, 1.2, 1.1, 1.2, 1.0, 1.3, 1.2], [1.4])
def testConfidenceScore_EmptySample(self):
# Confidence is zero if either or both samples are empty.
self._AssertConfidence(0.0, [], [])
self._AssertConfidence(0.0, [], [1.1, 1.2, 1.1, 1.2, 1.0, 1.3, 1.2, 1.3])
self._AssertConfidence(0.0, [1.1, 1.2, 1.1, 1.2, 1.0, 1.3, 1.2, 1.3], [])
def testConfidenceScore_FunctionalTestResults(self):
self._AssertConfidence(80.0, [1, 1, 0, 1, 1, 1, 0, 1], [0, 0, 1, 0, 1, 0])
self._AssertConfidence(99.9, [1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0])
def testConfidenceScore_RealWorldCases(self):
"""This method contains a set of data from actual bisect results.
The confidence scores asserted below were all copied from the actual
results, so the purpose of this test method is mainly to show what the
results for real cases are, and compare when we change the confidence
score function in the future.
"""
self._AssertConfidence(80, [133, 130, 132, 132, 130, 129], [129, 129, 125])
self._AssertConfidence(99.5, [668, 667], [498, 498, 499])
self._AssertConfidence(80, [67, 68], [65, 65, 67])
self._AssertConfidence(0, [514], [514])
self._AssertConfidence(90, [616, 613, 607, 615], [617, 619, 619, 617])
self._AssertConfidence(0, [3.5, 5.8, 4.7, 3.5, 3.6], [2.8])
self._AssertConfidence(90, [3, 3, 3], [2, 2, 2, 3])
self._AssertConfidence(0, [1999004, 1999627], [223355])
self._AssertConfidence(90, [1040, 934, 961], [876, 875, 789])
self._AssertConfidence(90, [309, 305, 304], [302, 302, 299, 303, 298])
def testCorrectlyFindsBreakingRange(self):
revision_states = self.mock_bisect_state.mock_revision_states
revision_states[0].passed = 0
revision_states[1].passed = 0
revision_states[2].passed = 1
revision_states[3].passed = 1
revision_states[4].passed = 1
results = BisectResults(self.mock_bisect_state, self.mock_depot_registry,
self.mock_opts, self.mock_warnings)
self.assertEqual(revision_states[2], results.first_working_revision)
self.assertEqual(revision_states[1], results.last_broken_revision)
def testCorrectlyComputesRegressionStatistics(self):
revision_states = self.mock_bisect_state.mock_revision_states
revision_states[0].passed = 0
revision_states[0].value = {'values': [1000, 999, 998]}
revision_states[1].passed = 0
revision_states[1].value = {'values': [980, 1000, 999]}
revision_states[2].passed = 1
revision_states[2].value = {'values': [50, 45, 55]}
revision_states[3].passed = 1
revision_states[3].value = {'values': [45, 56, 45]}
revision_states[4].passed = 1
revision_states[4].value = {'values': [51, 41, 58]}
results = BisectResults(self.mock_bisect_state, self.mock_depot_registry,
self.mock_opts, self.mock_warnings)
self.assertAlmostEqual(99.9, results.confidence)
self.assertAlmostEqual(1909.86547085, results.regression_size)
self.assertAlmostEqual(7.16625904, results.regression_std_err)
def testFindsCulpritRevisions(self):
revision_states = self.mock_bisect_state.mock_revision_states
revision_states[1].depot = 'chromium'
revision_states[2].depot = 'webkit'
results = BisectResults(self.mock_bisect_state, self.mock_depot_registry,
self.mock_opts, self.mock_warnings)
self.assertEqual(1, len(results.culprit_revisions))
self.assertEqual(('b', {'test': 'b'}, 'chromium'),
results.culprit_revisions[0])
def testFindsOtherRegressions(self):
revision_states = self.mock_bisect_state.mock_revision_states
revision_states[0].passed = 0
revision_states[0].value = {'values': [100, 100, 100]}
revision_states[1].passed = 0
revision_states[1].value = {'values': [100, 100, 100]}
revision_states[2].passed = 1
revision_states[2].value = {'values': [10, 10, 10]}
revision_states[3].passed = 1
revision_states[3].value = {'values': [100, 100, 100]}
revision_states[4].passed = 1
revision_states[4].value = {'values': [60, 60, 60]}
results = BisectResults(self.mock_bisect_state, self.mock_depot_registry,
self.mock_opts, self.mock_warnings)
expected_regressions = [[revision_states[2], revision_states[1], 99.9],
[revision_states[4], revision_states[3], 80.0]]
self.assertEqual(expected_regressions, results.other_regressions)
def testNoResultBasedWarningsForNormalState(self):
results = BisectResults(self.mock_bisect_state, self.mock_depot_registry,
self.mock_opts, self.mock_warnings)
self.assertEqual(0, len(results.warnings))
def testWarningForMultipleCulpritRevisions(self):
self.mock_bisect_state.mock_revision_states[2].passed = 'Skipped'
results = BisectResults(self.mock_bisect_state, self.mock_depot_registry,
self.mock_opts, self.mock_warnings)
self.assertEqual(1, len(results.warnings))
def testWarningForTooLowRetryLimit(self):
self.mock_opts.repeat_test_count = 1
results = BisectResults(self.mock_bisect_state, self.mock_depot_registry,
self.mock_opts, self.mock_warnings)
self.assertEqual(1, len(results.warnings))
def testWarningForTooLowConfidence(self):
revision_states = self.mock_bisect_state.mock_revision_states
revision_states[2].value = {'values': [95, 100, 90]}
revision_states[3].value = {'values': [95, 100, 90]}
revision_states[4].value = {'values': [95, 100, 90]}
results = BisectResults(self.mock_bisect_state, self.mock_depot_registry,
self.mock_opts, self.mock_warnings)
self.assertGreater(results.confidence, 0)
self.assertEqual(1, len(results.warnings))
def testWarningForZeroConfidence(self):
revision_states = self.mock_bisect_state.mock_revision_states
revision_states[2].value = {'values': [100, 105, 95]}
revision_states[3].value = {'values': [100, 105, 95]}
revision_states[4].value = {'values': [100, 105, 95]}
results = BisectResults(self.mock_bisect_state, self.mock_depot_registry,
self.mock_opts, self.mock_warnings)
self.assertEqual(0, results.confidence)
self.assertEqual(1, len(results.warnings))
if __name__ == '__main__':
unittest.main()
|
