diff options
| -rw-r--r-- | build/android/devil/android/decorators.py | 26 | ||||
| -rw-r--r-- | build/android/devil/android/decorators_test.py | 30 | ||||
| -rw-r--r-- | build/android/devil/android/sdk/adb_wrapper.py | 8 | ||||
| -rw-r--r-- | build/android/devil/utils/timeout_retry.py | 16 |
4 files changed, 71 insertions, 9 deletions
diff --git a/build/android/devil/android/decorators.py b/build/android/devil/android/decorators.py index b263790..004ac8bc 100644 --- a/build/android/devil/android/decorators.py +++ b/build/android/devil/android/decorators.py @@ -19,7 +19,9 @@ DEFAULT_TIMEOUT_ATTR = '_default_timeout' DEFAULT_RETRIES_ATTR = '_default_retries' -def _TimeoutRetryWrapper(f, timeout_func, retries_func, pass_values=False): +def _TimeoutRetryWrapper( + f, timeout_func, retries_func, retry_if_func=timeout_retry.AlwaysRetry, + pass_values=False): """ Wraps a funcion with timeout and retry handling logic. Args: @@ -50,7 +52,8 @@ def _TimeoutRetryWrapper(f, timeout_func, retries_func, pass_values=False): desc = '%s(%s)' % (f.__name__, ', '.join(itertools.chain( (str(a) for a in args), ('%s=%s' % (k, str(v)) for k, v in kwargs.iteritems())))) - return timeout_retry.Run(impl, timeout, retries, desc=desc) + return timeout_retry.Run(impl, timeout, retries, desc=desc, + retry_if_func=retry_if_func) except reraiser_thread.TimeoutError as e: raise device_errors.CommandTimeoutError(str(e)), None, ( sys.exc_info()[2]) @@ -75,6 +78,25 @@ def WithTimeoutAndRetries(f): return _TimeoutRetryWrapper(f, get_timeout, get_retries) +def WithTimeoutAndConditionalRetries(retry_if_func): + """Returns a decorator that handles timeouts and, in some cases, retries. + + 'timeout' and 'retries' kwargs must be passed to the function. + + Args: + retry_if_func: A unary callable that takes an exception and returns + whether failures should be retried. + Returns: + The actual decorator. + """ + def decorator(f): + get_timeout = lambda *a, **kw: kw['timeout'] + get_retries = lambda *a, **kw: kw['retries'] + return _TimeoutRetryWrapper( + f, get_timeout, get_retries, retry_if_func=retry_if_func) + return decorator + + def WithExplicitTimeoutAndRetries(timeout, retries): """Returns a decorator that handles timeouts and retries. diff --git a/build/android/devil/android/decorators_test.py b/build/android/devil/android/decorators_test.py index 1b6179e6..a00b128 100644 --- a/build/android/devil/android/decorators_test.py +++ b/build/android/devil/android/decorators_test.py @@ -82,6 +82,36 @@ class DecoratorsTest(unittest.TestCase): timeout=10, retries=1) self.assertEquals(exception_desc, str(e.exception)) + def testConditionalRetriesDecoratorRetries(self): + def do_not_retry_no_adb_error(exc): + return not isinstance(exc, device_errors.NoAdbError) + + actual_tries = [0] + + @decorators.WithTimeoutAndConditionalRetries(do_not_retry_no_adb_error) + def alwaysRaisesCommandFailedError(timeout=None, retries=None): + actual_tries[0] += 1 + raise device_errors.CommandFailedError('Command failed :(') + + with self.assertRaises(device_errors.CommandFailedError): + alwaysRaisesCommandFailedError(timeout=10, retries=10) + self.assertEquals(11, actual_tries[0]) + + def testConditionalRetriesDecoratorDoesntRetry(self): + def do_not_retry_no_adb_error(exc): + return not isinstance(exc, device_errors.NoAdbError) + + actual_tries = [0] + + @decorators.WithTimeoutAndConditionalRetries(do_not_retry_no_adb_error) + def alwaysRaisesNoAdbError(timeout=None, retries=None): + actual_tries[0] += 1 + raise device_errors.NoAdbError() + + with self.assertRaises(device_errors.NoAdbError): + alwaysRaisesNoAdbError(timeout=10, retries=10) + self.assertEquals(1, actual_tries[0]) + def testDefaultsFunctionDecoratorDoesTimeouts(self): """Tests that the defaults decorator handles timeout logic.""" DecoratorsTest._decorated_function_called_count = 0 diff --git a/build/android/devil/android/sdk/adb_wrapper.py b/build/android/devil/android/sdk/adb_wrapper.py index 85d46ea..38db0e1 100644 --- a/build/android/devil/android/sdk/adb_wrapper.py +++ b/build/android/devil/android/sdk/adb_wrapper.py @@ -64,6 +64,10 @@ def _FindAdb(): raise device_errors.NoAdbError() +def _ShouldRetryAdbCmd(exc): + return not isinstance(exc, device_errors.NoAdbError) + + DeviceStat = collections.namedtuple('DeviceStat', ['st_mode', 'st_size', 'st_time']) @@ -99,9 +103,9 @@ class AdbWrapper(object): cmd.extend(args) return cmd - # pylint: disable=unused-argument + #pylint: disable=unused-argument @classmethod - @decorators.WithTimeoutAndRetries + @decorators.WithTimeoutAndConditionalRetries(_ShouldRetryAdbCmd) def _RunAdbCmd(cls, args, timeout=None, retries=None, device_serial=None, check_error=True, cpu_affinity=None): # pylint: disable=no-member diff --git a/build/android/devil/utils/timeout_retry.py b/build/android/devil/utils/timeout_retry.py index 1d748a9..e950e11 100644 --- a/build/android/devil/utils/timeout_retry.py +++ b/build/android/devil/utils/timeout_retry.py @@ -125,8 +125,12 @@ def _LogLastException(thread_name, attempt, max_attempts, log_func): log_func('*' * 80) +def AlwaysRetry(_exception): + return True + + def Run(func, timeout, retries, args=None, kwargs=None, desc=None, - error_log_func=logging.critical): + error_log_func=logging.critical, retry_if_func=AlwaysRetry): """Runs the passed function in a separate thread with timeouts and retries. Args: @@ -138,6 +142,8 @@ def Run(func, timeout, retries, args=None, kwargs=None, desc=None, desc: An optional description of |func| used in logging. If omitted, |func.__name__| will be used. error_log_func: Logging function when logging errors. + retry_if_func: Unary callable that takes an exception and returns + whether |func| should be retried. Defaults to always retrying. Returns: The return value of func(*args, **kwargs). @@ -163,13 +169,13 @@ def Run(func, timeout, retries, args=None, kwargs=None, desc=None, logging.info('Still working on %s', desc if desc else func.__name__) else: return thread_group.GetAllReturnValues()[0] - except reraiser_thread.TimeoutError: + except reraiser_thread.TimeoutError as e: # Timeouts already get their stacks logged. - if num_try > retries: + if num_try > retries or not retry_if_func(e): raise # Do not catch KeyboardInterrupt. - except Exception: # pylint: disable=broad-except - if num_try > retries: + except Exception as e: # pylint: disable=broad-except + if num_try > retries or not retry_if_func(e): raise _LogLastException(thread_name, num_try, retries + 1, error_log_func) num_try += 1 |