import boto3 from labelaudit import config from labelaudit.consts import report_column_headings from labelaudit.logic import audit_persistence from labelaudit.logic import audit_prep_report from labelaudit.logic.stream_to_s3 import StreamToS3 HTML_EMAIL_BODY = """ Hello,

The requested Audit doc for Label {} has been completed. Please download here.""" TEXT_EMAIL_BODY = """ Hello, The requested Audit doc for Label {} has been completed. Please download it here: {}/youtubeaudit/vendorId/{} """ def generate(report): """Generate an audit report and write to s3. Updates the audit report status to complete on success. Args: report (dict): see labelaudit.tasks.report_generation_tasks """ youtube_audit_id = report.get('youtubeAuditId') limit = total = config.NUM_RELEASES_WRITE_CSV offset = 0 bucket_root = config.S3_BUCKET_ROOT or '' vendor_id = report.get('vendorId') s3_key = '{}{}/{}.csv'.format( bucket_root, vendor_id, youtube_audit_id) # Write location of file we are creating to the db # TODO abstract the path generation and move this back to # where we set status to 'generating' write_report_path(youtube_audit_id, config.S3_BUCKET, s3_key) # TODO log that we are starting to write a file to S3 with StreamToS3( s3_key, config.S3_BUCKET, report_column_headings.VAPI_REPORT_FIELDNAMES, report_column_headings.REPORT_COLUMN_HEADINGS) as s: s.write_column_headers() while offset < total: rows = audit_prep_report.fetch( youtubeAuditId=youtube_audit_id, offset=offset, limit=limit) if (rows.errors): raise Exception( 'Error getting report rows for generation: {}' .format(rows.errors)) total = rows.message.get('total_records') # stream results to S3 file one offset batch at a time s.write_batch_rows(rows.message.get('report_rows')) offset += limit audit_persistence.update_report_status(youtube_audit_id, 'complete') email_notification(vendor_id) def email_notification(vendor_id): """Email auditor that report is ready for download. Args: vendor_id (int): label id for the report """ conn = boto3.client('ses', region_name=config.AWS_REGION) conn.send_email( source=config.EMAIL_FROM_ADDRESS, subject='Vendor {} - Label Audit Prep Doc Completed'.format(vendor_id), body=TEXT_EMAIL_BODY.format(vendor_id, config.OA_URL, vendor_id), html_body=HTML_EMAIL_BODY.format(vendor_id, config.OA_URL, vendor_id), to_addresses=[config.EMAIL_TO_ADDRESS]) def write_report_path(youtube_audit_id, bucket, key): """Write storage path of the audit report to db via VAPI. Args: youtube_audit_id (int) bucket (str): s3 bucket without trailing slash key (str): s3 key """ full_s3_path = '{}/{}'.format(bucket, key) audit_persistence.update(youtube_audit_id, { 'reportLocation': full_s3_path})