"""Export processed digital sales to S3.""" import argparse import collections import gzip import logging import os import shutil import boto3 import botocore import MySQLdb from royalties import config logging.basicConfig( level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s') logging.getLogger('boto3').setLevel(logging.CRITICAL) logging.getLogger('botocore').setLevel(logging.CRITICAL) CliArgs = collections.namedtuple( 'CliArgs', [ 'overwrite', 'skip_unload', 'skip_compress' ] ) def parse_args(): """Parse command line arguments. Returns: CliArgs: table and other arguments. """ parser = argparse.ArgumentParser( description='fact_analytics sanity check') parser.add_argument( '-o', '--overwrite', action='store_true', default=False, help='Overwrite S3 file if exists') parser.add_argument( '-s', '--skip-unload', action='store_true', default=False, help=( 'Skip Mysql unload step. We assume a ' 'file has been unloaded already')) parser.add_argument( '-c', '--skip-compress', action='store_true', default=False, help='Compress the file being uploaded') args = parser.parse_args() return CliArgs( overwrite=args.overwrite, skip_unload=args.skip_unload, skip_compress=args.skip_compress) def s3_file_exists(bucket, key): """Check if s3 file exists. Args: bucket (str): s3 bucket. key (str): s3 key. """ s3 = boto3.resource('s3') try: s3.Object(bucket, key).load() except botocore.exceptions.ClientError as e: if e.response['Error']['Code'] == '404': return False else: raise return True def s3_upload_file(bucket_name, key_name, local_path): """Upload file to S3. Args: bucket_name (str): destination bucket name. key_name (str): destination s3 filename to create. local_path (str): local file to upload to s3. """ session = boto3.Session() s3_client = session.client('s3') s3_config = boto3.s3.transfer.TransferConfig() transfer = boto3.s3.transfer.S3Transfer(client=s3_client, config=s3_config) transfer.upload_file(local_path, bucket_name, key_name) def mysql_unload(period_id, outfile_path): """Unload mysql data locally. Args: period_id (int): accounting period id. outfile_path (str): path to file on mysql box. """ with open( os.path.realpath(os.path.dirname(__name__)) + '/queries/art_relations/export_processed_dig_sales.sql') as fd: export_sql = fd.read() db = MySQLdb.connect(**config.ar_config) cursor = db.cursor() cursor.execute(export_sql, { 'period_id': period_id, 'local_path': outfile_path }) cursor.close() db.close() def compress_file(source_path, dest_path): """Compress a file at source_path and write it to dest_path. The dest_path file will be overwritten if it exists. Args: source_path (str): local path to the file to compress. dest_path (str): local path to the compressed file. """ with open(source_path, 'rb') as file_in: if os.path.exists(dest_path): os.remove(dest_path) with gzip.open(dest_path, 'wb') as file_out: shutil.copyfileobj(file_in, file_out) def get_final_local_file_path(skip_compress=False): """Get the path to the final outfile. Checks that the local dir exists. It's assumed that this is mounted and originates from the mysql server's outfile directory because mysql_unload outputs to an OUTFILE. If compression is enabled, the file extension of the file will be updated accordingly. Returns: str: Path to the final outfile. """ output_name = '{filename}' if not skip_compress: output_name = '{filename}.gz' local_dir = os.environ.get('APP_FILE_DIR', False) if not local_dir: raise Exception('Var APP_FILE_DIR missing.') if not os.path.isdir(local_dir): raise Exception('App path missing: {path}'.format(path=local_dir)) return output_name.format(filename=config.app_file_path) def main(): """Export processed_dig_sales to an outfile.""" args = parse_args() if s3_file_exists(config.s3_bucket, config.s3_key): s3_path = 's3://{bucket}/{key}'.format( bucket=config.s3_bucket, key=config.s3_key) if not args.overwrite: logging.info( 'Destination file already exists. Exiting: %s', s3_path) return else: logging.info( 'Destination file already exists. Overwriting: %s', s3_path) local_filepath = get_final_local_file_path(args.skip_compress) if not args.skip_unload: if os.path.exists(config.app_file_path): os.remove(config.app_file_path) # Export data to outfile. logging.info('Unloading mysql data.') mysql_unload(config.period_id, config.mysql_outfile_path) logging.info('Successfully exported local file.') if not args.skip_compress: # GZIP export file. logging.info('Compressing export file.') compress_file(config.mysql_outfile_path, local_filepath) logging.info('Successfully compressed export file.') else: logging.info('Skipping unload. Using %s', local_filepath) logging.info('Uploading file to S3.') s3_upload_file(config.s3_bucket, config.s3_key, local_filepath) logging.info('Successfully uploaded file to S3.') if __name__ == '__main__': main()