"""Producer methods to put products data to the availability polling queue.""" import json from botocore import exceptions as botocore_exceptions from oto import response from sentry_sdk import capture_exception, capture_message from availability.connectors import loggly from availability.connectors import sqs from availability.constants import error from availability.constants import field_const from availability.constants import models from availability.logic import countries from availability.logic.queue import utils from availability.models import product_in_store from availability.models import task logger = loggly.get_current_logger() def convert_products_to_sqs_messages(sqs_queue, products, correlation_id): """Get products to poll as SQS messages. Args: sqs_queue (boto.sqs.queue.Queue): Instance of Amazon SQS. products (iterable): Iterable with dicts with 'product_in_store_id', 'orchard_product_id', 'store_id' and 'upc' keys. correlation_id (str): correlation_id from request header. Yields: tuple: SQS Message instance with product data and product_in_store_id tuples. """ for product_to_poll_dict in products: msg_attrs = { sqs.CORRELATION_ID_ATTRIBUTE: { 'DataType': 'String', 'StringValue': str(correlation_id), }, } message_body = json.dumps(product_to_poll_dict) msg = {'message_body': message_body, 'message_attributes': msg_attrs} yield msg, product_to_poll_dict[field_const.PRODUCT_IN_STORE_ID] def set_product_countries_list(products, correlation_id): """Set 'countries' key to list of countries where product is expected. This generator receives iterable of dicts where each dict contains 'countries' key. This key is comma-separated string of countries where product is already released. Generator uses this info to get list of countries where release is still expected and yields the copy of each dict with 'countries' key replaced by this list of countries. Skips and logs products for which we were not able to receive expected countries. Args: products (iterable): Iterable with dicts with 'product_in_store_id', 'orchard_product_id', 'store_id', 'upc' and 'countries' keys. correlation_id (str): correlation_id from request header. Yields: dict: the same dict as received in `products` arg but with 'countries' key set to list of countries where release is expected. """ logger.info( 'Setting products countries list', resources=dict(correlation_id=correlation_id)) for product in products: expected_countries = countries.get_expected_countries( product[field_const.UPC], product[field_const.COUNTRIES], product[field_const.STORE_ID], correlation_id) if not expected_countries: msg = 'Error getting expected countries for a product: ' logging_context = dict( correlation_id=correlation_id, errors=expected_countries.errors, product=product) logger.error(msg, resources=logging_context) # These errors are not fatal to the whole run, but quite important, # so we also send them to Sentry. capture_message(msg + str(logging_context)) continue yield dict(product, countries=expected_countries.message) def put_products_to_poll_messages_to_queue( *, store_id, correlation_id, **kwargs): """Put messages with product data to SQS queue. This function should be called from a request handler that does any unexpected exception handling. Individual failing SQS write operations are logged and corresponding products are skipped. Args: store_id (int): Store ID. correlation_id (str): correlation_id from request header. Returns: Response: empty successful response on success or fatal response on error during SQS queue access. """ logging_context = dict( store_id=store_id, correlation_id=correlation_id, **kwargs) logger.info('Putting messages into the queue', resources=logging_context) try: sqs_queue = utils.get_availability_queue(store_id) except ValueError: logger.error('Unable to get SQS queue', resources=logging_context) return response.create_error_response( code=error.ERROR_CODE_NO_SQS_QUEUE, message=error.ERROR_MESSAGE_NO_SQS_QUEUE) product_to_poll_dicts = product_in_store.products_to_poll( store_id, **kwargs) product_to_poll_dicts = set_product_countries_list( product_to_poll_dicts, correlation_id) logger.info('Converting products to messages', resources=logging_context) product_messages = convert_products_to_sqs_messages( sqs_queue, product_to_poll_dicts, correlation_id) logging_context = dict(message_number=0) for msg, product_in_store_id in product_messages: logging_context['message_number'] += 1 msg_body = msg.get('message_body') logging_context.update(dict( msg_body=msg_body, product_in_store_id=product_in_store_id)) try: sqs_queue.send_message( MessageBody=msg.get('message_body'), MessageAttributes=msg.get('message_attributes')) except botocore_exceptions.ClientError as e: # If there is a problem sending a single SQS message - # report it and continue. logging_context['exc_args'] = e.args logger.error( 'Failed to send a message', resources=logging_context) capture_exception(e) continue logger.info( 'Message written to queue', resources=logging_context) task.change_status( product_in_store_id, models.TASK_STATUS_IN_QUEUE) logger.info( 'Task status set to "in_queue"', resources=logging_context) logger.info( 'Written messages to SQS queue', resources=logging_context) return response.Response()