"""Misc utilities.""" from functools import wraps import logging import time import os LOGGING_LEVEL = os.environ.get('LOGGING_LEVEL', logging.INFO) log = logging.getLogger() log.setLevel(LOGGING_LEVEL) def dynamo_active_record_to_json(item, include_locked=False): """Convert DynamoDB record format to JSON. Args: item (dict): DynamoDB record include_locked (bool): should locked territories be included Returns: dict: JSON representation """ data = { 'isrc': item['isrc']['S'], 'territories': {} } for terr, tuid_map in item['territories']['M'].items(): if 'M' in tuid_map: tuid_map = {'L': [tuid_map]} tuids_list = [t['M']['tuid']['N'] for t in tuid_map['L']] data['territories'][terr] = tuids_list if include_locked: data['locked_territories'] = {} if item.get('locked_territories'): for terr, tuid_map in item['locked_territories']['M'].items(): data['locked_territories'][terr] = tuid_map['M']['reason']['S'] return data class RetryCountExceededError(Exception): """Raised by @retry decorator when it exceeds the defined retry count.""" def retry( error_condition=lambda err: True, retry_count=10, retry_timeout=1, progressive_timeout=True): """Retries a function call in case of exception. You could decorate any function or method with this if you need to repeatedly call this method a couple of times with an increasing interval in case of some error raised during the method call. Args: error_condition (callable(error)): Function that will check whether we should do retries for a particular error. E.g. you can check error class, some it's fields or values. retry_count (int): Number ot retries retry_timeout (int): Timeout in seconds to wait between retries progressive_timeout (bool): If True, the timeout value will be increased by 0.5 sec during each consecutive retry Raises: RetryCountExceededError: Error is being raised in case of retry count exceeded """ def wrapper(fn): @wraps(fn) def wrapped(*args, **kwargs): retries = 0 timeout = retry_timeout last_err = None while retries < retry_count: try: result = fn(*args, **kwargs) return result except Exception as err: if error_condition(err): last_err = err time.sleep(timeout) retries += 1 timeout += 0.5 if progressive_timeout else 0 else: raise err raise RetryCountExceededError(last_err) return wrapped return wrapper class S3UploadError(Exception): """Raised by upload_to_s3 in case of upload failure.""" @retry( error_condition=lambda err: type(err) is S3UploadError, progressive_timeout=False) def upload_to_s3(s3_client, bucket_name, s3_key, json_data): """Upload data to S3 bucket. Function is decorated with `retry` and will retry the upload in case of S3 upload failure. Args: s3_client (object): boto S3 client instance bucket_name (str): destination bucket name s3_key (str): S3 path to save the data json_data (str): json data serialized as string Raises: S3UploadError: in case of S3 upload failure """ try: s3_result = s3_client.put_object( Bucket=bucket_name, Key=s3_key, Body=bytearray(json_data, 'utf8')) except Exception as ex: raise S3UploadError(ex) if s3_result['ResponseMetadata']['HTTPStatusCode'] != 200: log.error('S3 upload failed: %s', s3_key) log.error('S3 response: %s', s3_result['ResponseMetadata']) raise S3UploadError(s3_result)