"""Vector order model.""" import sqlalchemy from src.configs import config from src.connectors import art_relations from src.connectors import direct_delivery from src.connectors import request from src.constants import fields from src.constants import services from src.models import sql_queries from src.utils import chunks def _extract_from_encoding_queue(db_record): """Extract necessary data from EQ DB record. Args: db_record (dict): EQ DB record Returns: dict: necessary fields in a proper format """ return { fields.ORDER_ID: db_record[fields.ENCODING_ORDER_ID], fields.ORDER_TYPE: db_record[fields.ENCODING_ORDER_TYPE], fields.PRIORITY: db_record[fields.PRIORITY], fields.META_UPDATE: db_record[fields.META_UPDATE] == 'Y', fields.ENCODER_ID: db_record[fields.ENCODER_ID] } def get_encoding_queue_bulk(encoding_queue_ids, change_field_names=True): """Get multiple encoding_queue records by encoding_queue_ids. Args: encoding_queue_ids (iterable): encoding_queue_id values. change_field_names (bool): change field names to be same as in Dynamo Returns: dict: A two-level dict with PK as a key and selected records in nested dicts. Example: { 10001: { 'encoding_order_id': 10011, 'encoding_order_type': 'release', ... }, 20002: { 'encoding_order_id': 20022, 'encoding_order_type': 'release', ... }, } """ if not encoding_queue_ids: return {} with direct_delivery.session_scope() as session: result = session.execute( sqlalchemy.text(sql_queries.DD_SELECT_ENCODING_QUEUE_BULK), {fields.SQL_ID_LIST: encoding_queue_ids}) result_dict = {} for row in result.mappings().all(): row_dict = dict(row) queue_id = row_dict[fields.ENCODING_QUEUE_ID] if change_field_names: row_dict = _extract_from_encoding_queue(row_dict) result_dict[queue_id] = row_dict return result_dict def get_encoding_order_db_bulk(order_ids): """Get create user and date from encoding_order by list of order_id. Args: order_ids (list): list of encoding_order_id (int) Returns: dict: key (int) encoding_order_id, value (dict): key (str) field name, value (object) EO field value """ with art_relations.session_scope() as session: result = session.execute( sqlalchemy.text(sql_queries.AR_SELECT_ENCODING_ORDER_BULK), {fields.SQL_ID_LIST: order_ids}) result_dict = {} for row in result.mappings().all(): result_dict[row[fields.ORDER_ID]] = { fields.CREATED_AT: row[fields.CREATED_AT], fields.USER_ID: row[fields.USER_ID] } return result_dict # The AWS EB microservice gateway responds with HTTP 502 for large query # strings. ~1.7K seems to be the limit, which gives us ~ 250 comma separated # 6-digit order IDs. # 100 should be a very safe choice. Longer input lists are only possible on # a very large Kinesis batch sizes during the mass updates (e.g. the backfill). @chunks.execute_in_chunks(max_chunk_size=100) def get_encoding_order_via_api_bulk(order_ids): """Get multiple Vector orders from DDB via single micro-service call. Example return value: { 1562059: {'user_id': 18, 'created_at': '2017-12-19T08:03:28'}, 1562000: {'user_id': 18, 'created_at': '2017-12-19T08:03:30'}, ... } Args: order_ids (iterable): Vector order IDs as integers or strings. Returns: dict: A two-level dict with PK as a key and a Vector orders records as nested dicts. """ order_ids_param = ','.join(map(str, order_ids)) result = request.get( services.OWS_VECTORORDER, f'/orders?{fields.ORDER_ID}={order_ids_param}') if not result: return {} selected_fields = fields.USER_ID, fields.CREATED_AT return { int(item[fields.ORDER_ID]): {f: item[f] for f in selected_fields} for item in result[fields.RESULT_ITEMS] } def get_encoding_order_bulk(order_ids): """Get create user and date from DDB or AR by order_id list. Args: order_ids (list): order_id (int) list Returns: dict: key (int) order_id, value (dict): key (str) field name, value (object) field value """ # Useful only during backfill. Can be left here safely until # 'Create Vector Orders' is complete, but better to remove it. if config.FETCH_VECTOR_ORDERS_FROM_AR_ONLY: return get_encoding_order_db_bulk(order_ids) orders = get_encoding_order_via_api_bulk(order_ids) not_found_order_ids = list(set(order_ids) - set(orders)) if not_found_order_ids: # Old Vector orders (before November 2017) are likely not to be present # in the DynamoDB table which backs the microservice endpoint. So, we # fetch them form AR DB here. orders_from_db = get_encoding_order_db_bulk(not_found_order_ids) orders.update(orders_from_db) return orders