""" Misc utility classes and functions ================================== """ import codecs from functools import wraps import json import threading import time from googleapiclient.errors import HttpError from ytownership import config from ytownership.errors import RetryCountExceededError class DotDict(dict): """Simple wrapper around dict to access its items using dot notation """ def __getattr__(self, item): """Get item from dictionary by it's key, using dot-notation Args: item (str): key Returns: value, or None if there is no item with such key """ return self.get(item) def __setattr__(self, key, value): """Set value in dictionary, using dot-notation Args: key (str): item key value: item value """ self[key] = value def retry(error_condition=lambda err: is_gapi_rate_limit(err), retry_count=config.API_RETRY_COUNT, retry_timeout=config.API_RETRY_TIMEOUT, progressive_timeout=True): """Decorator for retrying function call in case of exception You could decorate any function or method with this if you need to repeatedly call this method a couple of times with an increasing interval in case of some error raised during the method call. Args: error_condition (callable(error)): Function that will check whether we should do retries for a particular error. E.g. you can check error class, some it's fields or values. retry_count (int): Number ot retries retry_timeout (int): Timeout in seconds to wait between retries progressive_timeout (bool): If True, the timeout value will be increased by 0.5 sec during each consecutive retry Raises: RetryCountExceededError: Error is being raised in case of retry count exceeded """ def wrapper(fn): @wraps(fn) def wrapped(*args, **kwargs): retries = 0 timeout = retry_timeout while retries < retry_count: try: result = fn(*args, **kwargs) return result except Exception as err: if error_condition(err): time.sleep(timeout) retries += 1 timeout += 0.5 if progressive_timeout else 0 else: raise err raise RetryCountExceededError return wrapped return wrapper def is_gapi_rate_limit(err): """Checks if passed error is HttpError for GAPI rate limit exceeded Args: err (Exception): instance of raised exception Returns: bool: Whether exception is HttpError and reason contains information, that user rate limit was exceeded during Google API call """ if type(err) is not HttpError: return False rate_reason = 'userRateLimitExceeded' try: error = json.loads(codecs.utf_8_decode(err.content)[0]).get('error') return error and (error['code'] == 403 and any(e['reason'] == rate_reason for e in error['errors'])) except: # noqa return False def rate_limited(max_per_second): """Decorator for throttling API requests. Args: max_per_second (int): maximum amount of request per second. Returns: Wrapped function. """ lock = threading.Lock() min_interval = 1.0 / max_per_second def decorate(func): last_time_called = time.perf_counter() @wraps(func) def rate_limited_function(*args, **kwargs): try: lock.acquire() nonlocal last_time_called elapsed = time.perf_counter() - last_time_called left_to_wait = min_interval - elapsed if left_to_wait > 0: time.sleep(left_to_wait) ret = func(*args, **kwargs) last_time_called = time.perf_counter() return ret finally: try: lock.release() except RuntimeError: pass return rate_limited_function return decorate def create_http_error(http_status, error_reason): return HttpError( DotDict({'status': http_status}), codecs.utf_8_encode(json.dumps({'reason': error_reason}))[0])