"""Util functions for AWS s3.""" import csv from datetime import datetime from datetime import timedelta from os import path import re from boto.s3.connection import S3Connection from boto.s3.key import Key import boto3 from flows.theatrical import config as conf from flows.theatrical import utils def _try_parse(bucket, prefix, s3_key): """Try to extract full file descriptor. In case of failure (with incorrect file) returns empty dict. Args: bucket (str): s3 bucket name. prefix (str): s3 key prefix. s3_key (str): s3 object key. Returns: dict: Dict with file descriptor. """ filename_pattern = re.compile( prefix + '(?P[0-9]{8})-(?P[0-9]{8})-Theatrical\.csv') filename = path.basename(s3_key) res = filename_pattern.match(s3_key) if res: start = res.groupdict().get('start') end = res.groupdict().get('end') date_start = datetime.strptime(start, conf.DATE_FORMAT).date() date_end = datetime.strptime(end, conf.DATE_FORMAT).date() if (date_start.weekday(), date_end.weekday()) in [(0, 3), (4, 6)]: return { 's3_path': s3_key, 'filename': filename, 'bucket_name': bucket, 'date_start': utils.serialize_date(date_start), 'date_end': utils.serialize_date(date_end), 'batch_date': utils.serialize_date( date_start + timedelta(days=5))} return {} def find_files(bucket_name, prefix, date_start, date_end): """Find paths for files on s3. Args: bucket_name (str): s3 bucket name. prefix (str): path to searching folder. date_start (date): start bound of date range. date_end (date): end excluding bound of date range. Returns: list: List of dicts that contains files info. """ files = [] bucket = boto3.resource('s3').Bucket(bucket_name) for o in bucket.objects.filter(Prefix=prefix): file = _try_parse(bucket_name, prefix, o.key) if not file: # skip not appropriate files continue file_date_start = utils.deserialize_date(file['date_start']) file_date_end = utils.deserialize_date(file['date_end']) if file_date_start < date_end and file_date_end >= date_start: files.append(file) return files def _to_snake_case(string): """Convert to snake case.""" return string.strip().lower().replace(' ', '_') def download_csv(bucket_name, key): """Download CSV file from s3 bucket. Args: bucket_name (str): bucket name. key (str): s3 object key. Returns: tuple: Header and list of rows. """ obj = boto3.resource('s3').Object(bucket_name, key) data = obj.get()['Body'].read() lines = data.decode().splitlines() raw_header, *rows = [row[:-1] for row in csv.reader(lines)] header = [_to_snake_case(col) for col in raw_header] return header, rows def move_files(files): """Move files to the appropriate locations. Args: files (list(dict)): list of files for moving. Each element of list should have this structure e.g.: {'old_bucket': 'old_bucket_name', 'old_key_path': '/old/key/path/file.txt', 'new_bucket': 'new_bucket_name', 'new_dir': '/new/dir/', 'file_name': 'file_name.csv'} """ s3_connection = S3Connection() for file in files: new_bucket_object = s3_connection.get_bucket(file['new_bucket']) new_key = Key(new_bucket_object) new_key.key = '{}{}'.format(file['new_dir'], file['file_name']) new_bucket_object.copy_key( new_key, file['old_bucket'], file['old_key_path']) old_bucket_object = s3_connection.get_bucket(file['old_bucket']) old_key = Key(old_bucket_object) old_key.key = file['old_key_path'] old_bucket_object.delete_key(old_key)