"""The S3-related Garcon tasks and helpers.""" import datetime import gzip import tempfile import time from boto.s3.connection import Bucket from boto.s3.connection import S3Connection from boto.s3.key import Key import boto3 from garcon import task from feed_sender.garcon_contrib.aws.utils import garcon_s3 as s3utils def read_s3_object(s3_path, encoding=None): """Read small S3 object into string. @todo(paulo): moving this to garcon contrib later on. Args: s3_path (bytes or str): Full S3 path. encoding (str): Charater encoding value. Returns: bytes or str: Contents of the file. """ bucket, path = s3utils.extract_bucket_path(s3_path) bucket_obj = Bucket(S3Connection(), bucket) key = Key(bucket_obj, path) return key.get_contents_as_string(encoding=encoding) def _list_directories(s3_path): """List all S3 object with prefix /. @todo(paulo): moving this to garcon contrib later on. Args: s3_path (str): Full S3 path to the object(s) Returns: list: List of 'folder' names (or objects with / as last character). """ bucket, path = s3utils.extract_bucket_path(s3_path) bucket_obj = Bucket(S3Connection(), bucket) path = '{}/'.format(path.rstrip('/')) return [v.name for v in bucket_obj.list(path, '/')] def check_s3_key_exist(s3_key_path): """Check if S3 object exists in the specific path. Args: s3_key_path (str): Full path to the S3 key. Return: boolean: True or False of whether it exist. """ bucket_name, key_path = s3utils.extract_bucket_path(s3_key_path) file_bucket = Bucket(S3Connection(), bucket_name) keys = [key.name for key in file_bucket.list(key_path)] return len(keys) > 0 @task.decorate(timeout=7000) def wait_for_s3_objects_existence( activity, s3_path, wait_time, check_time_interval): """Wait for S3 object in s3_path to exist. Check if object exists within a specified period of time. Args: activity (ActivityWorker): The SWF activity worker. s3_path (str): S3 path to object to check. wait_time (int): Time in second(s) to wait for the S3 object to exists. After this time, this task will return False. check_time_interval (int): Time in second(s) for each iteration to wait before issuing another status check request. Returns: dict: {'exists': True} or {'exists': False}. """ wait_time = wait_time or 10 check_time_interval = check_time_interval or 1 time_expire = datetime.datetime.today() + datetime.timedelta( seconds=wait_time) found = False while time_expire > datetime.datetime.today(): activity.logger.info('Checking if done file exists: ({})...'.format( s3_path)) if check_s3_key_exist(s3_path): found = True break time.sleep(check_time_interval) return {'exists': found} def is_gzip_file_on_s3_corrupted(s3_path, logger=None): """Check if gzip file on S3 is corrupted. Args: s3_path (str): Full S3 path. logger (activity logger): Optional injection of logger object. Returns: Boolean: True if corrupted, False otherwise. """ with tempfile.NamedTemporaryFile() as fp: corrupted = False try: bucket_name, key_path = s3utils.extract_bucket_path(s3_path) boto3.resource('s3').meta.client.download_file( bucket_name, key_path, fp.name) with gzip.open(fp.name) as g: while g.read(1024 * 1024): pass except Exception as e: if logger: logger.info(str(e)) corrupted = True return corrupted