""" Copy AVRO files for QA script. ============================== Copy Avro files from production S3 bucket to QA environment. Example call: copy_avro_for_qa -p 202,203,204 -i quarter copy_avro_for_qa -p 202 -i month copy_avro_for_qa -p 202 -i month -s System Requirements: 1. ReadOnly access on bucket "prod-statement-detail-exports" 2. WriteOnly access on bucket "qa-statement-detail-exports" """ import argparse import boto3 from processing_accounting.conf import settings s3 = boto3.resource('s3') client = boto3.client('s3') def copy_avro_files(periods, payment_interval, account_ids, source_key_path): """Copy avro files from source location to destination location Args: periods (str): comma delimited period ids. Example: 202,203,204 payment_interval (str): month or quarter. account_ids (list): list of integer label ids. source_key_path (str): path to the virtual directory on source s3 key path. """ bucket = s3.Bucket(settings.PRODUCTION_BUCKET) for account_id in account_ids: for obj in bucket.objects.filter( Prefix=source_key_path.format( periods, payment_interval, account_id)): print('copying: {}'.format(obj.key)) s3.meta.client.copy( {'Bucket': settings.PRODUCTION_BUCKET, 'Key': obj.key}, settings.QA_BUCKET, obj.key) def run(*args): """Main entry point of this cli script. Args: args (tuple): user parameters: -p or -i. """ parser = argparse.ArgumentParser(description='Command line util') parser.add_argument( '-p', '--period_ids', required=True, help=( 'Desired accounting period ids' 'Example: copy_avro_for_qa -p \'202,203,204\'')) parser.add_argument( '-s', '--subaccount', action='store_true', help=( 'Use for subaccount.' 'Example: copy_avro_for_qa -p \'202,203,204\' -s')) parser.add_argument( '-i', '--payment_interval', required=True, help=( 'Desired payment interval. Either: "month" or "quarter"' 'Example: copy_avro_for_qa -p \'212\' -i month'), choices=['month', 'quarter']) args = parser.parse_args(args) if args else parser.parse_args() args = args or '{}' account_ids = settings.MONTHLY_LABEL_IDS if args.payment_interval == 'quarter': account_ids = settings.QUARTERLY_LABEL_IDS key_path = settings.LABEL_KEY_PATH if args.subaccount: key_path = settings.SUBACCOUNT_KEY_PATH account_ids = settings.MONTHLY_SUBACCOUNT_IDS if args.payment_interval == 'quarter': account_ids = settings.QUARTERLY_SUBACCOUNT_IDS copy_avro_files( args.period_ids, args.payment_interval, account_ids, key_path)