1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
|
#!/usr/bin/env python
# Copyright 2014 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Runs a test repeatedly to measure its flakiness. The return code is non-zero
if the failure rate is higher than the specified threshold, but is not 100%."""
import argparse
import multiprocessing.dummy
import subprocess
import sys
import time
def load_options():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--retries', default=1000, type=int,
help='Number of test retries to measure flakiness.')
parser.add_argument('--threshold', default=0.05, type=float,
help='Minimum flakiness level at which test is '
'considered flaky.')
parser.add_argument('--jobs', '-j', type=int, default=1,
help='Number of parallel jobs to run tests.')
parser.add_argument('command', nargs='+', help='Command to run test.')
return parser.parse_args()
def run_test(job):
print 'Starting retry attempt %d out of %d' % (job['index'] + 1,
job['retries'])
return subprocess.check_call(job['cmd'], stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
def main():
options = load_options()
num_passed = num_failed = 0
running = []
pool = multiprocessing.dummy.Pool(processes=options.jobs)
args = [{'index': index, 'retries': options.retries, 'cmd': options.command}
for index in range(options.retries)]
results = pool.map(run_test, args)
num_passed = len([retcode for retcode in results if retcode == 0])
num_failed = len(results) - num_passed
if num_passed == 0:
flakiness = 0
else:
flakiness = num_failed / float(len(results))
print 'Flakiness is %.2f' % flakiness
if flakiness > options.threshold:
return 1
else:
return 0
if __name__ == '__main__':
sys.exit(main())
|
