"""Module that responds for input message processing.""" import json import re import sentry_sdk from ddtrace import tracer from oto import response from notifications_delivery.config import ( EMAIL_SENDER, ENVIRONMENT, ENVIRONMENT_PROD, GET_STREAM_FEED_GROUPS, ) from notifications_delivery.connectors import loggly, ses from notifications_delivery.constants import error from notifications_delivery.constants.exceptions import IgnoredMessageError from notifications_delivery.constants.notifications import ( AWAL_COMPANY_BRAND, CONTENT_REVIEW_CONFIRMATION_EMAIL, CONTENT_REVIEW_TESTING_EMAIL, DEFAULT_LANGUAGE, MESSAGE_TYPE_BOUNCE, MESSAGE_TYPE_COMPLAINT, MESSAGE_TYPE_GENERAL, MESSAGE_TYPE_NOTIFICATION, ORCHARD_COMPANY_BRAND, PROD_EMAIL_SENDERS, QA_EMAIL_SENDERS, TEMPLATES_WITH_FEATURE_FLAG, ) from notifications_delivery.models import ows_account, ows_features, ows_notifications, ows_users from notifications_delivery.utils import ( email_utils, message_detector, notification_extractor, template_renderer, ) logger = loggly.get_current_logger() def process_message(message): """Process a message coming from the SQS queue. Args: message (sqs.Message): message pulled from SQS-queue. Returns: Response: successful response in case there was no exceptions. """ try: try: message_body = json.loads(message.body) message_type = message_detector.get_message_type(message_body) message_format = 'orchard' except json.JSONDecodeError: message_body = message.body message_type = MESSAGE_TYPE_NOTIFICATION message_format = 'stream' if tracer.enabled: span = tracer.current_span() span.set_tag('message.type', message_type) span.set_tag('message.format', message_format) logger.info( 'SQS message is picked up for processing', resources={ 'message_body': message_body, 'message_type': message_type, 'message_format': message_format } ) if message_type == MESSAGE_TYPE_NOTIFICATION: return process_notification(message_body, message_format) if message_type == MESSAGE_TYPE_GENERAL: return process_general_notification(message_body) elif message_type == MESSAGE_TYPE_BOUNCE: return process_bounce(message_body) elif message_type == MESSAGE_TYPE_COMPLAINT: return process_complaint(message_body) else: raise Exception(error.ERROR_MESSAGE_INVALID_MESSAGE) except IgnoredMessageError as e: return response.create_error_response( code=error.ERROR_CODE_IGNORED_MESSAGE, message=str(e) ) except Exception as e: sentry_sdk.capture_exception(e) return response.create_error_response( code=error.ERROR_CODE_MESSAGE_PROCESSING, message=str(e)) @tracer.wrap() def process_notification(message, fmt): """Process a message body coming from a notification. Args: message (dict): message body pulled from SQS-message. fmt (str): object or stream Returns: Response: successful response in case there was no exceptions. """ notification = notification_extractor.parse_message(message, fmt) if not notification: raise Exception(error.ERROR_MESSAGE_INVALID_MESSAGE) users_info = notification.get('users_info') default_brand = notification['payload'].get('default_brand') if tracer.enabled: span = tracer.current_span() span.set_tag('feed_id', notification['feed_id'].lower()) span.set_tag('feed_name', notification['feed_name']) span.set_tag('brand', default_brand) logger.info( f'Received notification to send out for: ' f' users_info: {notification.get("users_info")},' f' user_ids: {notification.get("user_ids")},' f' user_emails: {notification.get("user_emails")},' f' feed_name: {notification.get("feed_name")},' f' feed_id: {notification.get("feed_id")},' f' template: {notification.get("template")}', resources={ 'feed_id': notification['feed_id'].lower(), 'feed_name': notification['feed_name'] } ) if not users_info: users_info = [] user_ids = notification.get('user_ids') if user_ids: users_info_response = ows_users.get_users_info(user_ids) users_info = users_info_response.message user_emails = notification.get('user_emails') if user_emails: for user_email in user_emails: users_info.append({ 'user_id': user_email, 'email': user_email }) if not users_info: raise Exception(error.ERROR_MESSAGE_NO_USERS) label_type = notification['feed_id'] if 'vendor' in label_type: label_id = label_type.split('_')[1] if label_id: valid_vendor = is_valid_vendor(label_id) if not valid_vendor: for user_info in users_info: unsubscribe_alw_users(user_info['email']) unsubscribe_email_from_all_feeds(user_info['email']) return response.create_error_response( code=error.ERROR_CODE_MESSAGE_PROCESSING, message=f'Invalid or inactive vendor: {label_id}' ) email_utils.format_notification_for_rendering(notification) users_info = remove_inactive_users(users_info) users_info = remove_duplicate_users(users_info) logger.info( f'Ready to send out emails for notification: ' f' users_info: {users_info},' f' feed_name: {notification.get("feed_name")},' f' feed_id: {notification.get("feed_id")},' f' template: {notification.get("template")}', resources={ 'feed_id': notification['feed_id'].lower(), 'feed_name': notification['feed_name'] } ) for user_info in users_info: is_user_email_valid = is_valid_email(user_info['email']) if not is_user_email_valid: unsubscribe_alw_users(user_info['email']) unsubscribe_email_from_all_feeds(user_info['email']) continue user_locale = user_info.get('language', DEFAULT_LANGUAGE) if 'account' in user_info and 'vendor_id' in user_info['account'] and ( ows_account.get_vendor_company_brand( user_info['account']['vendor_id']).message == AWAL_COMPANY_BRAND): company_brand = AWAL_COMPANY_BRAND else: company_brand = ORCHARD_COMPANY_BRAND template_ = f'{company_brand}_{notification["template"]}' if EMAIL_SENDER: email_sender = EMAIL_SENDER elif ENVIRONMENT == ENVIRONMENT_PROD: email_sender = PROD_EMAIL_SENDERS[template_] else: email_sender = QA_EMAIL_SENDERS[template_] if ENVIRONMENT == ENVIRONMENT_PROD: content_review_cc_recipient = CONTENT_REVIEW_CONFIRMATION_EMAIL else: content_review_cc_recipient = CONTENT_REVIEW_TESTING_EMAIL email_subject = email_utils.get_email_subject( notification['template'], notification['payload'], user_locale) notification['payload']['preview_text'] = email_utils. \ get_preview_text( notification['template'], notification['payload'], user_locale) notification['payload']['unsubscribe_url'] = email_utils. \ generate_unsubscribe_url( user_info['user_id'], notification['feed_name'], notification['feed_id']) notification['payload']['identify_url'] = email_utils. \ generate_identify_url(user_info) notification['payload']['tracking_url'] = email_utils. \ generate_tracking_url( user_info['user_id'], notification['feed_name'], notification['feed_id'], email_subject) email_template = notification['template'] if email_template in TEMPLATES_WITH_FEATURE_FLAG: user_features = ows_features.get_user_features(user_info['user_id']) if email_template == 'digital_approval': if company_brand == ORCHARD_COMPANY_BRAND: email_template = 'theorchard_digital_approval' if email_template == 'spike_detector': if user_features.get('notifications_spike_detector_new_email') == 'enabled': email_template = 'branded_spike_detector' if email_template == 'analytics_digest': if user_features.get('notifications_analytics_digest_new_email') == 'enabled': email_template = 'branded_analytics_digest' if tracer.enabled: span = tracer.current_span() span.set_tag('email', user_info['email']) span.set_tag('template_name', email_template) rendered_html = template_renderer.render_template( email_template, notification['payload'], user_locale) logger.info( f'Send email to {user_info["email"]} with topic {email_subject} that has' f' default_brand={default_brand}' f' and label_name={notification["payload"].get("label_name")}', resources={ 'feed_id': notification['feed_id'].lower(), 'feed_name': notification['feed_name'], 'email': user_info['email'], 'template_name': email_template } ) ses.send_email({ 'subject': email_subject, 'to': [user_info['email']], 'sender': email_sender, 'html_body': rendered_html, 'text_body': '' }) email_utils.track_email_sent( user_info['user_id'], notification['feed_name'], notification['feed_id']) # send follow-up email for content review confirmation if email_template in ( 'digital_approval', 'digital_rejection', 'theorchard_digital_approval' ): logger.info( f'Send carbon copy of email to {content_review_cc_recipient}' f' with topic {email_subject} that has' f' default_brand={default_brand}' f' and label_name={notification["payload"].get("label_name")}', resources={ 'feed_id': notification['feed_id'].lower(), 'feed_name': notification['feed_name'], 'template_name': email_template } ) ses.send_email({ 'subject': email_subject, 'to': [content_review_cc_recipient], 'sender': email_sender, 'html_body': rendered_html, 'text_body': '' }) return response.Response() @tracer.wrap() def process_bounce(message_body): """Process a message body coming from a bounce. Args: message_body (dict): message body pulled from SQS-message. Returns: Response: successful response in case there was no exceptions. """ emails = email_utils. \ extract_bounced_emails_from_ses_notification(message_body) if emails: logger.info( f'Extract bounced emails from ses-notifications that has message body:' f' {json.dumps(message_body)}', ) for email in emails: email_utils.track_email_bounce(email) # To re-enable if needed # unsubscribe_alw_users(email) return response.Response() @tracer.wrap() def process_complaint(message_body): """Process a message body coming from a complaint. Args: message_body (dict): message body pulled from SQS-message. Returns: Response: successful response in case there was no exceptions. """ emails = email_utils.extract_complaint_emails_from_ses_notification(message_body) if emails: logger.info( f'Process coming from a complaint a message body: {json.dumps(message_body)}', ) for email in emails: email_utils.track_email_complaint(email) return response.Response() @tracer.wrap() def process_general_notification(message): """Process a general notification. { "type": "GeneralNotification", "subject": "Host Read Ad Assigned {name}", "subject_variables": {"name": "Bob"}, "to": ["ratoui@theorchard.com"], "template": "host_read_ad_assigned", "lang": "en", "sender": '"Sony Music Podcasts" ', "variables": { "organization": "orchard", "from": "networkadmin@sonymusic.com", "podcast_title": "[PROD] Muschio Selvaggio", "episode_title": "Ep.14 Questa รจ la volta buona", "episode_url": "https://podcast.qaorch.com/podcast/6536/episode/3322", } } :param message: :return: """ user_locale = message['lang'] translations = email_utils.get_translations(user_locale) subject = translations.gettext(message['subject']) if 'subject_variables' in message: subject = subject.format(**message['subject_variables']) to = message['to'] template_name = message['template'] sender = message['sender'] user_id = message.get('userId') or message.get('user_id') if user_id: message['variables']['tracking_url'] = \ email_utils.generate_generic_tracking_url(user_id, subject) template = template_renderer.render_template( template_name, message['variables'], user_locale) with tracer.trace('send_generic_email_notification') as span: span.set_tag('email', to) span.set_tag('template_name', template_name) logger.info( f'Send email to {to} from {sender} with topic {subject} ' f'that has template {template_name}', resources={ 'email': to, 'template_name': template_name } ) ses.send_email({ 'subject': subject, 'to': to, 'sender': sender, 'html_body': template, 'text_body': '' }) if user_id: email_utils.track_generic_email_sent(user_id, subject) return response.Response() @tracer.wrap() def remove_inactive_users(users_info): """Remove inactive users from list. Args: users_info [dict]: users information. Returns: [dict]: users information without inactive users. """ active_users = [] for user_info in users_info: if 'active' not in user_info: active_users.append(user_info) else: if user_info['active'] == 'Y': active_users.append(user_info) return active_users @tracer.wrap() def remove_duplicate_users(users_info): """Remove duplicate users from list. Args: users_info [dict]: users information. Returns: [dict]: users information without duplicate users. """ unique_users = [] user_emails = [] for user_info in users_info: if user_info['email'] not in user_emails: user_emails.append(user_info['email']) unique_users.append(user_info) return unique_users @tracer.wrap() def unsubscribe_alw_users(email): """Unsubscribe alw users. Note: This function will soon be deprecated. Don't use it. Args: email str: user email. """ alw_users_info_response = ows_users.get_alw_users_by_email(email) if not alw_users_info_response: return alw_users_info = alw_users_info_response.message for user_info in alw_users_info['items']: for feed_group in GET_STREAM_FEED_GROUPS: subscriptions_response = ows_notifications.get_vendor_subscriptions( user_info['vendor_id'], feed_group) if not subscriptions_response: return is_subscribed = is_user_subscribed( subscriptions_response.message['items'], user_info['user_id']) if is_subscribed: logger.warning( f'Unsubscribe: Using old subscription system for: {email}', resources=user_info ) ows_notifications.unsubscribe_user(user_info, feed_group) def is_user_subscribed(subscriptions, user_id): """Check if user is already subscribed. Args: subscriptions (dict): vendor subscriptions. user_id (int): the user ID """ alw_user_id = 'alw_{user_id}'.format(user_id=user_id) is_subscribed = False for subscription in subscriptions: if alw_user_id in subscription: is_subscribed = True return is_subscribed def is_valid_vendor(vendor_id): """Check if vendor has a valid status. Args: vendor_id: the vendor id. """ vendor_info_response = ows_account.get_vendor_info(vendor_id) if not vendor_info_response: return True vendor_info = vendor_info_response.message if vendor_info['status'] != 'signed': return False return True def is_valid_email(user_email): """Check if email is valid. Args: user_email: the user email. """ email = user_email.replace(' ', '') if not email: return False if re.match(r'[^@]+@[^@]+\.[^@]+', email): return True else: return False @tracer.wrap() def unsubscribe_email_from_all_feeds(email): """Unsubscribe this email from all feeds that are handled by this daemon. This is equivalent to turning the toggle OFF for all email options from settings app. Args: email (str): email for identity. """ result = ows_users.get_identity_by_email(email) if not result or not result.message or not result.message.get('id'): # ideally users that are not in our system should not get emails. # Log this to see if these are valid use case and how to unsubscribe them. logger.error(f'Unsubscribe: Identity not found for this email: {email}') return result identity_data = result.message for feed_group in GET_STREAM_FEED_GROUPS: ows_notifications.user_subscription_toggle_off(identity_data['id'], feed_group)