import logging from apollo_utils.core.constants.dsp import DSP from typing import Any, Dict, Iterable, List, Sequence, Tuple, Union from server.constants.distributors import DISTRIBUTORS, DISTRIBUTORS_ORDER, IsSonyInclude from server.utils.common import prefix_str logger = logging.getLogger(__name__) def sme_response( data: Union[List[str], Tuple[List[str], Dict[str, Dict[str, str]]]], by_track: bool = False, with_id_to_idx: bool = False, remove_track_to_upc: bool = False, track_id_prefix: str = DSP.SPOTIFY.value, track_id_delimiter: str = "_", ) -> Union[List[dict], Tuple[List[dict], Dict[str, int]]]: """Format raw is-sony response. :param data: raw is-sony data, allowed to be in 2 formats, depending on 'by_track' parameter value: 1)by_track=False - list of upc 2)by_track=True - tuple of 2 items: list of track_id and dict of extra_data, containing track_id to upc dict. This format is a result of 'tracks_is_sony' scenario, check the scenario for more details. :param by_track: if True, data is tuple of 2 items: list of track_id and dict of track_id to upc, else data is list of upc. :param with_id_to_idx: if True, return tuple of 2 items: list of response dicts and dict of track_id/upc to their index in the response list, if False return only list of response dicts. :param remove_track_to_upc: if True, remove track_id to upc mapping from extra_data. :param track_id_prefix: track_id prefix to add to raw track_id, set to None to avoid adding prefix. :param track_id_delimiter: delimiter between track_id prefix and raw track_id. """ result, extra_data, id_to_idx = [], {}, {} if by_track and data: track_id_list, extra_data = data getter = extra_data.pop if remove_track_to_upc else extra_data.get track_id_to_upc = getter(IsSonyInclude.TRACK_UPC_MAPPING, {}) for i, track_id in enumerate(track_id_list): full_track_id = prefix_str(track_id, prefix=track_id_prefix, delimiter=track_id_delimiter) if with_id_to_idx: id_to_idx[full_track_id] = i result.append( { "upc": track_id_to_upc.get(track_id), "track_id": full_track_id, "distributed_by": DISTRIBUTORS.SME.value, } ) else: for i, upc in enumerate(data): if with_id_to_idx: id_to_idx[upc] = i result.append( { "upc": upc, "track_id": None, "distributed_by": DISTRIBUTORS.SME.value, } ) if with_id_to_idx: extra_data["track_id_to_idx"] = id_to_idx return result, extra_data def log_unknown_items( non_sme_result: dict, sme_ids: Sequence[str], requested_ids: Sequence[str], sme_by_track: bool = False ): """Use to detailed log unknown upcs and track_ids that were not resolved by Delphi and Apollo both.""" id_keys, unknowns, idx = ("unknown_upcs", "unknown_track_ids"), [], int(sme_by_track) for i, k in enumerate(id_keys): v = set(non_sme_result.get(k) or []) - set(sme_ids if i == idx else []) unknowns.append(v) if unknowns[idx]: log_prefix = "[log_unknown_items]" logger.warning( f"{log_prefix}[{len(unknowns[idx])}/{len(requested_ids)}] {id_keys[idx]} were not resolved.\n" f"{log_prefix}[{len(requested_ids)}] requested: {requested_ids}\n" f"{log_prefix}[{len(sme_ids)}] sme-related: {sme_ids}\n" f"{log_prefix}[{len(unknowns[1])}] unknown track ids: {unknowns[1]}\n" f"{log_prefix}[{len(unknowns[0])}] unknown upcs: {unknowns[0]}" ) def merge_response( data: Sequence[Any], sme: bool = True, non_sme: bool = True, sme_by_track: bool = False, distributors: set[str] = None, include: Iterable[str] = None, requested_ids: Sequence[str] = None, ) -> Tuple[List[dict], Dict[str, Any]]: """Merge distributors responses. Contains extra checks for multiple distributors, that are added because we are not sure about the data quality. Filters response items by 'distributors' if passed. Resolve cases when multiple distributors return the same track_id/upc according to the priority. Log all data issues. Later we can remove this logic and just combine two responses without handling their intersections. :param data: list of distributors responses, various length and content, depending on (non)-sme flags. :param sme: if True, sme-data is presented in the data list. :param non_sme: if True, non-sme-data is presented in the data list. :param sme_by_track: if True, sme-data is defined by track_id, else by upc. :param distributors: set of distributors to check, if None, check is not performed. :param include: List of additional data to include in extra_data. Available options are IsSonyInclude. :param requested_ids: list of requested track_ids or upcs, used for logging only. :return: Tuple of List of items { "upc": str, "track_id": Optional[str], _ "distributed_by: str (lowercase) } and extra_data dict. """ sme_data = data[0] if sme else [] non_sme_data = data[int(sme)] if non_sme else {} data_keys = ("upc", "track_id") id_key = data_keys[int(sme_by_track)] collisions = {} # multiple distributors for the same track_id/upc result, extra_data = sme_response( sme_data, by_track=sme_by_track, with_id_to_idx=True, remove_track_to_upc=IsSonyInclude.TRACK_UPC_MAPPING not in include, ) sme_id_to_idx = extra_data.pop("track_id_to_idx", {}) log_unknown_items(non_sme_data, sme_id_to_idx.keys(), requested_ids, sme_by_track=sme_by_track) for item in non_sme_data.get("items") or []: sme_item = None id_value = item[id_key] if not id_value: logger.warning(f"[merge_response] received empty {id_key} in item: {item}, skipped.") continue sme_idx = sme_id_to_idx.get(id_value) if sme_idx is not None: sme_item = result[sme_idx] distributed_by = [d.lower() for d in item.get("distributed_by", []) if d] + ( [sme_item["distributed_by"]] if sme_item else [] ) if distributors: distributed_by = list(set(distributed_by) & distributors) if not distributed_by: logger.warning( f"[merge_response] received no allowed {distributors} distributors in item: " f"{item}, skipped." ) continue if len(distributed_by) > 1: distributed_by.sort(key=lambda x: DISTRIBUTORS_ORDER.index(DISTRIBUTORS(x))) logger.warning( f"[merge_response][collision] received more than one distributor in item: {item};\n" f" distributors: {distributed_by} resolved to {distributed_by[0]}." ) collision_key = ",".join(distributed_by) collisions[collision_key] = collisions.get(collision_key, 0) + 1 item["distributed_by"] = distributed_by[0] if sme_item: sme_item["distributed_by"] = item["distributed_by"] continue item.pop("country_code", None) result.append(item) if collisions: logger.warning(f"[merge_response][collisions] statistics: {collisions}") return result, extra_data