import csv from datetime import datetime import os import threading import uuid import config import util def save_csv_report(s3_client, records, seq_id): if config.UPLOAD_IN_SEPARATE_THREAD: threading.Thread( target=generate_and_upload_report, args=(s3_client, records, seq_id,)).start() else: generate_and_upload_report(s3_client, records, seq_id) @util.retry(retry_count=3) def generate_and_upload_report(s3_client, records, seq_id): job_id = records[0]['job_id'] rnd = str(uuid.uuid1()) file_name = '{}_{}.csv'.format(rnd, seq_id) print('Saving CSV and uploading to S3. {}'.format(file_name)) file_path = os.path.join('/tmp', file_name) with open(file_path, 'w') as csvfile: writer = csv.writer(csvfile, delimiter=',') for record in records: row = [record['isrc'], record['territory'], record['tuid']] writer.writerow(row) date = datetime.utcnow().strftime('%Y-%m-%d') key = os.path.join(date, job_id, file_name) s3_client.upload_file(file_path, config.REPORTS_BUCKET, key) try: os.unlink(file_path) except: print('Failed to remove file: {}'.format(file_path)) full_path = os.path.join(config.REPORTS_BUCKET, key) print('{} uploaded to S3'.format(full_path))