"""Misc helper functions.""" import json import threading from _codecs import utf_8_decode from functools import wraps import time import psycopg2 from googleapiclient.errors import HttpError class RetryCountExceededError(Exception): """Raised by @retry decorator when it exceeds the defined retry count.""" def __str__(self): """Get string representation of an error.""" return 'Method retry count exceeded' def retry(error_condition=lambda err: is_psycopg_error(err), retry_count=10, retry_timeout=1, progressive_timeout=True): """Decorator for retrying function call in case of exception. You could decorate any function or method with this if you need to repeatedly call this method a couple of times with an increasing interval in case of some error raised during the method call. Args: error_condition (callable(error)): Function that will check whether we should do retries for a particular error. E.g. you can check error class, some it's fields or values. retry_count (int): Number ot retries retry_timeout (int): Timeout in seconds to wait between retries progressive_timeout (bool): If True, the timeout value will be increased by 0.5 sec during each consecutive retry Raises: RetryCountExceededError: Error is being raised in case of retry count exceeded """ def wrapper(fn): @wraps(fn) def wrapped(*args, **kwargs): retries = 0 timeout = retry_timeout while retries < retry_count: try: result = fn(*args, **kwargs) return result except Exception as err: if error_condition(err): time.sleep(timeout) retries += 1 timeout += 0.5 if progressive_timeout else 0 else: raise err raise RetryCountExceededError return wrapped return wrapper def is_psycopg_error(err): """Check if passed error is DB error. Args: err (Exception): instance of raised exception Returns: bool: Whether exception is Postgress DB error """ return type(err) is psycopg2.Error def rate_limited(max_per_second): """Decorator for throttling API requests. Args: max_per_second (int): maximum amount of request per second. Returns: Wrapped function. """ lock = threading.Lock() min_interval = 1.0 / max_per_second def decorate(func): last_time_called = time.perf_counter() @wraps(func) def rate_limited_function(*args, **kwargs): lock.acquire() nonlocal last_time_called elapsed = time.perf_counter() - last_time_called left_to_wait = min_interval - elapsed if left_to_wait > 0: time.sleep(left_to_wait) ret = func(*args, **kwargs) last_time_called = time.perf_counter() lock.release() return ret return rate_limited_function return decorate def is_gapi_rate_limit(err): """Checks if passed error is HttpError for GAPI rate limit exceeded Args: err (Exception): instance of raised exception Returns: bool: Whether exception is HttpError and reason contains information, that user rate limit was exceeded during Google API call """ if type(err) is not HttpError: return False rate_reason = 'userRateLimitExceeded' try: error = json.loads(utf_8_decode(err.content)[0]).get('error') return error and (error['code'] == 403 and any(e['reason'] == rate_reason for e in error['errors'])) except: return False