import boto3
from labelaudit import config
from labelaudit.consts import report_column_headings
from labelaudit.logic import audit_persistence
from labelaudit.logic import audit_prep_report
from labelaudit.logic.stream_to_s3 import StreamToS3
HTML_EMAIL_BODY = """
Hello,
The requested Audit doc for Label {} has
been completed. Please download
here."""
TEXT_EMAIL_BODY = """
Hello, The requested Audit doc for Label {} has
been completed. Please download it here:
{}/youtubeaudit/vendorId/{}
"""
def generate(report):
"""Generate an audit report and write to s3.
Updates the audit report status to complete on success.
Args:
report (dict): see labelaudit.tasks.report_generation_tasks
"""
youtube_audit_id = report.get('youtubeAuditId')
limit = total = config.NUM_RELEASES_WRITE_CSV
offset = 0
bucket_root = config.S3_BUCKET_ROOT or ''
vendor_id = report.get('vendorId')
s3_key = '{}{}/{}.csv'.format(
bucket_root, vendor_id, youtube_audit_id)
# Write location of file we are creating to the db
# TODO abstract the path generation and move this back to
# where we set status to 'generating'
write_report_path(youtube_audit_id, config.S3_BUCKET, s3_key)
# TODO log that we are starting to write a file to S3
with StreamToS3(
s3_key, config.S3_BUCKET,
report_column_headings.VAPI_REPORT_FIELDNAMES,
report_column_headings.REPORT_COLUMN_HEADINGS) as s:
s.write_column_headers()
while offset < total:
rows = audit_prep_report.fetch(
youtubeAuditId=youtube_audit_id, offset=offset, limit=limit)
if (rows.errors):
raise Exception(
'Error getting report rows for generation: {}'
.format(rows.errors))
total = rows.message.get('total_records')
# stream results to S3 file one offset batch at a time
s.write_batch_rows(rows.message.get('report_rows'))
offset += limit
audit_persistence.update_report_status(youtube_audit_id, 'complete')
email_notification(vendor_id)
def email_notification(vendor_id):
"""Email auditor that report is ready for download.
Args:
vendor_id (int): label id for the report
"""
conn = boto3.client('ses', region_name=config.AWS_REGION)
conn.send_email(
source=config.EMAIL_FROM_ADDRESS,
subject='Vendor {} - Label Audit Prep Doc Completed'.format(vendor_id),
body=TEXT_EMAIL_BODY.format(vendor_id, config.OA_URL, vendor_id),
html_body=HTML_EMAIL_BODY.format(vendor_id, config.OA_URL, vendor_id),
to_addresses=[config.EMAIL_TO_ADDRESS])
def write_report_path(youtube_audit_id, bucket, key):
"""Write storage path of the audit report to db via VAPI.
Args:
youtube_audit_id (int)
bucket (str): s3 bucket without trailing slash
key (str): s3 key
"""
full_s3_path = '{}/{}'.format(bucket, key)
audit_persistence.update(youtube_audit_id, {
'reportLocation': full_s3_path})