| Matthew Treinish | 0db5377 | 2013-07-26 10:39:35 -0400 | [diff] [blame] | 1 | # vim: tabstop=4 shiftwidth=4 softtabstop=4 |
| 2 | |
| 3 | # Copyright 2011 OpenStack Foundation. |
| 4 | # Copyright 2012, Red Hat, Inc. |
| 5 | # |
| 6 | # Licensed under the Apache License, Version 2.0 (the "License"); you may |
| 7 | # not use this file except in compliance with the License. You may obtain |
| 8 | # a copy of the License at |
| 9 | # |
| 10 | # http://www.apache.org/licenses/LICENSE-2.0 |
| 11 | # |
| 12 | # Unless required by applicable law or agreed to in writing, software |
| 13 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT |
| 14 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the |
| 15 | # License for the specific language governing permissions and limitations |
| 16 | # under the License. |
| 17 | |
| 18 | """ |
| 19 | Exception related utilities. |
| 20 | """ |
| 21 | |
| 22 | import logging |
| 23 | import sys |
| 24 | import time |
| 25 | import traceback |
| 26 | |
| 27 | from tempest.openstack.common.gettextutils import _ # noqa |
| 28 | |
| 29 | |
| 30 | class save_and_reraise_exception(object): |
| 31 | """Save current exception, run some code and then re-raise. |
| 32 | |
| 33 | In some cases the exception context can be cleared, resulting in None |
| 34 | being attempted to be re-raised after an exception handler is run. This |
| 35 | can happen when eventlet switches greenthreads or when running an |
| 36 | exception handler, code raises and catches an exception. In both |
| 37 | cases the exception context will be cleared. |
| 38 | |
| 39 | To work around this, we save the exception state, run handler code, and |
| 40 | then re-raise the original exception. If another exception occurs, the |
| 41 | saved exception is logged and the new exception is re-raised. |
| 42 | |
| 43 | In some cases the caller may not want to re-raise the exception, and |
| 44 | for those circumstances this context provides a reraise flag that |
| 45 | can be used to suppress the exception. For example: |
| 46 | |
| 47 | except Exception: |
| 48 | with save_and_reraise_exception() as ctxt: |
| 49 | decide_if_need_reraise() |
| 50 | if not should_be_reraised: |
| 51 | ctxt.reraise = False |
| 52 | """ |
| 53 | def __init__(self): |
| 54 | self.reraise = True |
| 55 | |
| 56 | def __enter__(self): |
| 57 | self.type_, self.value, self.tb, = sys.exc_info() |
| 58 | return self |
| 59 | |
| 60 | def __exit__(self, exc_type, exc_val, exc_tb): |
| 61 | if exc_type is not None: |
| 62 | logging.error(_('Original exception being dropped: %s'), |
| 63 | traceback.format_exception(self.type_, |
| 64 | self.value, |
| 65 | self.tb)) |
| 66 | return False |
| 67 | if self.reraise: |
| 68 | raise self.type_, self.value, self.tb |
| 69 | |
| 70 | |
| 71 | def forever_retry_uncaught_exceptions(infunc): |
| 72 | def inner_func(*args, **kwargs): |
| 73 | last_log_time = 0 |
| 74 | last_exc_message = None |
| 75 | exc_count = 0 |
| 76 | while True: |
| 77 | try: |
| 78 | return infunc(*args, **kwargs) |
| 79 | except Exception as exc: |
| 80 | if exc.message == last_exc_message: |
| 81 | exc_count += 1 |
| 82 | else: |
| 83 | exc_count = 1 |
| 84 | # Do not log any more frequently than once a minute unless |
| 85 | # the exception message changes |
| 86 | cur_time = int(time.time()) |
| 87 | if (cur_time - last_log_time > 60 or |
| 88 | exc.message != last_exc_message): |
| 89 | logging.exception( |
| 90 | _('Unexpected exception occurred %d time(s)... ' |
| 91 | 'retrying.') % exc_count) |
| 92 | last_log_time = cur_time |
| 93 | last_exc_message = exc.message |
| 94 | exc_count = 0 |
| 95 | # This should be a very rare event. In case it isn't, do |
| 96 | # a sleep. |
| 97 | time.sleep(1) |
| 98 | return inner_func |