"""Force takedown using SRR feed + ISRC.""" import argparse import json import random import time from datetime import datetime from datetime import timedelta from datetime import timezone import boto3 # https://github.com/theorchard/lambda-sound-recordings/blob/master/common/connectors/sftp.py # noqa:E501 import sftp def main(): """Entrypoint.""" args = _parse_args() # get SFTP RSA key from secrets manager secrets_client = boto3.client('secretsmanager') response = secrets_client.get_secret_value(SecretId=args.secret_arn) sftp_key = response['SecretString'] sftp_conn = sftp.Connection(**{ 'hostname': args.sftp_server, 'port': args.sftp_port, 'username': args.sftp_user, 'pkey': sftp_key }) response = secrets_client.get_secret_value(SecretId='prod/lambda-sr-delivery-meta/META_SFTP_AWS_CREDENTIALS') aws_args = json.loads(response['SecretString']) aws_access_key_id= aws_args['aws_access_key_id'] aws_secret_access_key=aws_args['aws_secret_access_key'] # require explicit test flag to avoid accidents meta_s3_client = boto3.client('s3') if args.s3_test else boto3.client('s3', aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key) meta_bucket = 'qa-sr-interm-delivery-bucket' if args.s3_test else 'orchard-audio-delivery' base_dir = 'fingerprinting' if args.send_to_s3 else '/' + args.sftp_dir # read in xml template template = None with open('template.xml', 'r') as f: template = f.read() # read in data from file, each line has one ISRC isrcs = [] with open(args.filename, 'r') as f: isrcs = [x for x in f.read().split('\n') if x] # filter out data if we want to start from middle if args.since is not None: isrcs = isrcs[isrcs.index(args.since) + 1:] # chunk ISRCs into batches batches = [ isrcs[x:x + args.batch_size] for x in range(0, len(isrcs), args.batch_size) ] # process one batch at a time for batch in batches: for isrc in batch: # generate XML created_timestamp = datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ') # noqa:E501 batch_id = _generate_batch_id() end_datetime = datetime.now() - timedelta(days=2) srr_xml = template.format( message_create_time=created_timestamp, message_id=batch_id, track_isrc=isrc, end_date=end_datetime.date() ) # format data into files files = [ ( bytes(srr_xml.encode('utf-8')), f'{base_dir}/{batch_id}/{isrc}/{isrc}.xml' ), ( b'', f'{base_dir}/{batch_id}/BatchComplete_{batch_id}.xml' ) ] # upload files (directories auto created) if args.send_to_s3: for data, filepath in files: print(f'uploading to s3: {filepath}') try: meta_s3_client.put_object( Bucket=meta_bucket, Key=filepath, Body=data, ContentType='application/xml' ) except Exception as e: print(f'Error uploading {filepath} to S3: {e}. Stopping') exit(-1) else: print('uploading to sftp') sftp_conn.upload_files(files) # debugging and useful to restart using "--since" print(isrc, batch_id) # pause execution time.sleep(args.sleep) # close connection sftp_conn.close() exit(0) # https://github.com/theorchard/lambda-sound-recordings/blob/master/lambda/sr-delivery-meta/src/app.py#L167 def _generate_batch_id() -> str: date_str = datetime.now().strftime('%Y%m%d%H%M%S') random_str = str(random.randint(0, 9999)).rjust(4, '0') return date_str + random_str def _parse_args(): """Read in CLI args.""" parser = argparse.ArgumentParser( formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument( 'filename', help='filename containing ISRCs' ) parser.add_argument( '--secret-arn', type=str, default='arn:aws:secretsmanager:us-east-1:437795906767:secret:prod/lambda-sr-delivery-meta/META_SFTP_PKEY-bNPPQU', # noqa:E501 help='AWS secrets manager ARN of meta SFTP key' ) parser.add_argument( '--sftp-server', type=str, default='s-eaeb3091ded4453fa.server.transfer.us-west-2.amazonaws.com', help='Meta SFTP server' ) parser.add_argument( '--sftp-user', type=str, default='orchard', help='Meta SFTP username' ) parser.add_argument( '--sftp-port', type=int, default=22, help='Meta SFTP port' ) parser.add_argument( '--sftp-dir', type=str, default='fingerprinting', help='Meta SFTP directory to upload into' ) parser.add_argument( '--batch-size', type=int, default=100, help='Uploads to process before sleeping' ) parser.add_argument( '--sleep', type=int, default=10, help='Seconds to sleep between batches' ) parser.add_argument( '--since', type=str, default=None, help='Start processing records after this ISRC' ) parser.add_argument( '--send-to-s3', type=bool, default=False, help='Use to send to S3, not SFTP' ) parser.add_argument( '--s3-test', type=bool, default=True, help='Use to send to PDE S3 as a test. Requires explicit override to send to production' ) return parser.parse_args() if __name__ == '__main__': main()