import copy from celery import chord from oto import response from masters_registry.constant import bulk_tasks_const from masters_registry.constant import error from masters_registry.constant import field_const from masters_registry.constant import opcode_const from masters_registry.logic import bulk_tasks as logic_bulk_tasks from masters_registry.logic import conflicts from masters_registry.logic import dms_carveout from masters_registry.logic.masters_registry import _is_valid_territories from masters_registry.models import bulk_tasks from masters_registry.models import locks as lock_model from masters_registry.models import ownership from masters_registry.models import yt_ownership def lock_territories_with_conflicts( lock_reason, isrc, territories_to_lock, correlation_id, user): """Lock a list of territories for single ISRC Args: lock_reason (str): reason for locking isrc (str): international standard recording codes territories_to_lock (list(str)): list of ISO-3166-1 or fully-qualified territories correlation_id (str): The correlation id for logging the microservice instance user (str): The user id for audit table logging Returns: response.Response """ validate_response, active_isrcs = _validate_isrcs_and_territories( [isrc], territories_to_lock ) if not validate_response: return validate_response else: territories_to_lock = validate_response.message active_isrc = active_isrcs[0] original_isrc = copy.deepcopy(active_isrc) lock_result = lock_model.lock_territories( active_isrc, territories_to_lock, lock_reason, correlation_id, user) lock_update_yt_claimed_territories( active_isrcs=[original_isrc], territories_to_remove={isrc: lock_result[opcode_const.REMOVE]}, correlation_id=correlation_id ) # Determine resolved conflicts existing_territories = original_isrc['territories'] updated_ownership = ownership.get_ownership(isrc) resolved_conflict = conflicts.determine_resolved_conflict( existing_territories, updated_ownership.message[field_const.TERRITORIES]) if resolved_conflict: ownership.insert_audit_record( opcode=opcode_const.CONFLICT_RESOLVED, isrc=isrc, territories=territories_to_lock, correlation_id=correlation_id, user=user, source=field_const.MANUAL_EDIT_ISRC, conflict=resolved_conflict) lock_response = { 'lock_reason': lock_reason, 'isrc': isrc, 'territories': territories_to_lock } # Append lock history history_response = ownership.get_lock_history_for_single_reason( lock_reason, isrc) if not history_response: return history_response lock_response.update(history_response.message) return response.Response(lock_response) def bulk_lock_territories( lock_reason, isrcs, territories_to_lock, correlation_id, user): """Bulk lock a list of ISRC/territories, Locks every combination of the ISRC and territory Args: lock_reason (str): reason for locking isrcs (list(str)): list of international standard recording codes territories_to_lock (list(str)): list of ISO-3166-1 or fully-qualified territories correlation_id (str): The correlation id for logging the microservice instance user (str): The user id for audit table logging Returns: response.Response """ # This local import is to avoid circular references in celery tasks from masters_registry.tasks import bulk as bulk_celery_tasks validate_response, active_isrcs = _validate_isrcs_and_territories( isrcs, territories_to_lock) if not validate_response: return validate_response else: territories_to_lock = validate_response.message to_remove, to_unlock, to_lock = _check_territories( active_isrcs, territories_to_lock, lock_reason) active_records = _lock_create_active_table_records( to_remove, to_unlock, to_lock, lock_reason) task_id = logic_bulk_tasks.create_task( correlation_id, user, bulk_tasks_const.BULK_LOCK, len(isrcs), isrcs) sorted_isrcs = conflicts.filter_lock_territories_without_conflict( active_isrcs, territories_to_lock) import_response = { 'lock_reason': lock_reason, 'accepted_isrcs': sorted_isrcs.get('accepted_isrcs', []) } lock_territories_with_conflict = sorted_isrcs.get('conflicting_isrcs') if lock_territories_with_conflict: import_response['conflicting_isrcs'] = ( lock_territories_with_conflict) if not active_records: result = _create_lock_result(isrcs, territories_to_lock, lock_reason) bulk_tasks.update_task(task_id, bulk_tasks_const.DONE_STATUS, result) return response.Response(import_response) bulk_celery_tasks.bulk_lock_isrcs.delay( lock_reason, isrcs, sorted(territories_to_lock), correlation_id, user, task_id ) return response.Response(import_response) def lock_isrc( isrc, isrc_record, active_record, territories_to_remove, territories_to_unlock, territories_to_lock, correlation_id, user, lock_territories=None): """Function performs lock for single ISRC It updates active table, audit table and yt Args: isrc (str): international standard recording code isrc_record (dict): ISRC active table record active_record (dict): records for updating active table territories_to_remove (list): list of territories to remove territories_to_unlock (list): list of territories to unlock territories_to_lock (list): list of territories to lock correlation_id (str): correlation id for logging user (str): The user id for audit table logging lock_territories (list(str)): Territories that were received by handler Returns: response.Response: result of lock """ return lock_isrc_with_internal_conflict( isrc, isrc_record, active_record, lock_territories, user, correlation_id) def lock_isrc_with_internal_conflict( isrc, isrc_record, active_record, lock_territories, user, correlation_id): """Lock territory for ISRC accounting for possible internal confilct. Args: isrc (str): international standard recording code isrc_record (dict): ISRC active table record active_record (dict): records for updating active table lock_territories (list(str)): Territories that were received by handler user (str): The user id for audit table logging correlation_id (str): correlation id for logging Returns: response.Response: result of lock """ _, conflicting_territories = ( conflicts._split_isrc_territories_by_internal_conflict(isrc_record)) checked_lock_territores = [] territories_with_conflict = [] for territory in lock_territories: if territory in conflicting_territories: territories_with_conflict.append(territory) else: checked_lock_territores.append(territory) lock_reason = active_record['lock_reason'] to_remove, to_unlock, to_lock = _check_territories( [isrc_record], checked_lock_territores, lock_reason) territories = { 'to_lock': list(to_lock[isrc]), 'to_unlock': list(to_unlock[isrc]), 'to_remove': list(to_remove[isrc]) } to_remove = (to_remove, opcode_const.REMOVE) to_unlock = (to_unlock, opcode_const.UNLOCK) to_lock = (to_lock, opcode_const.LOCK) ownership.update_active_record( isrc, territories, lock_reason=lock_reason) lock_update_audit_table( to_remove, to_unlock, to_lock, correlation_id, user, lock_reason) failed_isrcs = lock_update_yt_claimed_territories( [isrc_record], to_remove[0], correlation_id) success_territories = checked_lock_territores if not failed_isrcs else [] failed_territories = ( territories_with_conflict if not failed_isrcs else lock_territories) response_payload = { 'territories': success_territories, 'failed_territories': failed_territories } if failed_isrcs: return response.create_error_response( error.DMS_CARVEOUT_ERROR, message=response_payload) if territories_with_conflict: return response.create_error_response( error.LOCK_ISRC_ERROR_CODE, message=response_payload) return response.Response() def _create_lock_result(isrcs, territories, lock_reason): """Helper function for creating result dict Args: lock_reason (str): reason for locking isrcs (list(str)): list of international standard recording codes territories (list(str)): list of ISO-3166-1 or fully-qualified territories Returns: dict """ result = dict() result['successful_isrcs'] = isrcs result['failed_isrcs'] = [] result['territories'] = territories result['reason'] = lock_reason return result def lock_update_audit_table( to_remove, to_unlock, to_lock, correlation_id, user, lock_reason, isrcs_to_skip=()): """Helper function for updating audit table Args: to_remove (tuple): containing dict with ISRCs and set of territories to_unlock (tuple): containing dict with ISRCs and set of territories to_lock (tuple): containing dict with ISRCs and set of territories correlation_id (str): correlation id for logging user (str): user id for audit table logging lock_reason (str): reason for locking isrcs_to_skip (list): list of ISRCs that will be skipped """ records = [] for territories, code in (to_remove, to_unlock, to_lock): if code == opcode_const.LOCK: reason = lock_reason else: reason = None records.extend( _create_records( territories, correlation_id, user, code, reason=reason, isrcs_to_skip=isrcs_to_skip ) ) ownership.update_multiple_ownerships(records) def _get_ownership_info_by_isrc(active_isrcs, isrc): """Find the ownership record for a particular ISRC in a list of records Args: active_isrcs (list): list of ownership records from active table of MR DynamoDB isrc (str): ISRC of the record that we want to find Returns: dict: ownership record """ for isrc_info in active_isrcs: if isrc_info[field_const.ISRC] == isrc: return isrc_info return None def lock_update_yt_claimed_territories( active_isrcs, territories_to_remove, correlation_id, isrcs_to_skip=()): """Helper function for updating youtube ownership Args: active_isrcs (set): existing ISRCs territories_to_remove (dict): dict with ISRCs and set of territories correlation_id (str): correlation id for logging isrcs_to_skip (list): list of ISRCs that will be skipped """ failed_isrcs = [] updated_claimed_territories = _get_updated_claimed_territories( active_isrcs, territories_to_remove) for isrc, claimed_territories in updated_claimed_territories.items(): if isrc not in isrcs_to_skip: if len(claimed_territories) > 0: ownership_info = _get_ownership_info_by_isrc( active_isrcs, isrc) carveout_result = dms_carveout.ownership_check_dms_carveout( ownership_info, list(claimed_territories), correlation_id) if not carveout_result or not carveout_result.message: failed_isrcs.append(isrc) else: allowed_territories = carveout_result.message yt_ownership.send_message( isrc, allowed_territories, correlation_id) else: yt_ownership.send_message( isrc, claimed_territories, correlation_id) return failed_isrcs def _validate_isrcs_and_territories(isrcs, territories): """Helper function for validating isrcs and territories Args: isrcs (list(str)): list of international standard recording codes territories (list(str)): list of ISO-3166-1 or fully-qualified territories Returns: tuple: containing error_response and None if territories or isrcs are invalid or Response() and active_isrcs if territories and isrcs are valid """ territories_response = _is_valid_territories(territories) if not territories_response: return territories_response, None else: territories = territories_response.message active_isrcs = ownership.get_existing_isrcs_in_active_table(isrcs) isrcs_response = _is_valid_isrcs(isrcs, active_isrcs) if not isrcs_response: return isrcs_response, None return response.Response(territories), active_isrcs def _lock_create_active_table_records(to_remove, to_unlock, to_lock, reason): """Helper function to create records for updating active table Args: to_remove (dict): dict with ISRCs and set of ISO-3166-1 territories to remove returned by _check_territories function to_unlock (dict): dict with ISRCs and set of ISO-3166-1 territories to unlock returned by _check_territories to_lock (dict): dict with ISRCs and set of ISO-3166-1 territories to lock returned by _check_territories function reason (str): lock reason Returns: list(dict) """ isrcs = {} territories_keys = ( ('to_remove', to_remove), ('to_unlock', to_unlock), ('to_lock', to_lock), ) for key, territories_data in territories_keys: for isrc, territories in territories_data.items(): if territories: if isrc not in isrcs: isrcs[isrc] = {} isrcs[isrc][key] = list(territories) records = [] for isrc, territories in isrcs.items(): records.append({ 'isrc': isrc, 'territories': territories, 'lock_reason': reason }) return records def _get_updated_claimed_territories(active_isrcs, territories_to_remove): """For every ISRC determines list of claimed territories Args: active_isrcs (list(dict)): list of ownership info dictionaries territories_to_remove (dict): dict with ISRCs and set of territories returned by _check_territories Returns: dict: Dictionary containing ISRCs with set of claimed territories Example: { {'AAAA': {'AS', 'AF'}}, {'BBBB': {'CA', 'DE'}} } """ updated_claimed_territories = {} for isrc_object in active_isrcs: isrc = isrc_object[field_const.ISRC] to_remove = territories_to_remove.get(isrc) if to_remove: claimed = set(isrc_object[field_const.TERRITORIES].keys()) updated_claimed_territories[isrc] = claimed.difference(to_remove) return updated_claimed_territories def _check_territories(active_isrcs, territories, lock_reason): """Checks which territories need to be removed, unlocked and locked Args: active_isrcs (list(dict)): list of ownership info dictionaries territories (list(str)): list of ISO-3166-1 territories lock_reason (str): lock reason Returns: tuple: Tuple containing three tuples with territories that need to be unlocked, removed and locked. Each tuple contains: dict with ISRCs and set of ISO-3166-1 territories operation code Example: ( ({'AAAAA': {'US' 'CA'}}, 'REMOVE'), ({'AAAAA': {'GB' 'DE'}}, 'UNLOCK'), ({'AAAAA': {'PL' 'UA'}}, 'LOCK'), ) """ territories_to_lock = {} territories_to_unlock = {} territories_to_remove = {} for isrc_object in active_isrcs: isrc = isrc_object[field_const.ISRC] to_lock = set(territories) locked_territories = isrc_object[field_const.LOCKED_TERRITORIES] locked = set(locked_territories.keys()) to_unlock = locked.intersection(territories) territories_to_unlock[isrc] = set() for territory in to_unlock: reason = locked_territories[territory][field_const.REASON] if reason != lock_reason: territories_to_unlock[isrc].add(territory) else: to_lock.remove(territory) claimed = set(isrc_object[field_const.TERRITORIES].keys()) territories_to_remove[isrc] = claimed.intersection(territories) territories_to_lock[isrc] = to_lock return ( territories_to_remove, territories_to_unlock, territories_to_lock, ) def _create_records( items, correlation_id, user, opcode, reason=None, isrcs_to_skip=()): """Creates list of masters_audit table records Args: items (dict): isrc and list of ISO-3166-1 territories correlation_id (str): correlation id user (str): user opcode (str): operation code reason (str): lock reason isrcs_to_skip (list): list of ISRCs that will be skipped Returns: list of masters_audit table records """ records = [] for isrc, territories in items.items(): if isrc not in isrcs_to_skip and territories: record = ownership.create_audit_record( opcode, isrc, list(territories), correlation_id, user, reason=reason ) records.append(record) return records def _is_valid_isrcs(isrcs, active_isrcs): """Checks if given ISRCs are valid Checks if all ISRCs exist in art_relations.track table and in active table Returns: response.Response """ isrcs = set(isrcs) existing_isrcs = ownership.get_existing_isrcs_in_tracks(isrcs) missing_isrcs = isrcs.difference(existing_isrcs) if missing_isrcs: return response.create_error_response( error.NOT_EXISTING_ISRCS_IN_TRACK, list(missing_isrcs)) existing_isrcs = {isrc['isrc'] for isrc in active_isrcs} missing_isrcs = isrcs.difference(existing_isrcs) if missing_isrcs: return response.create_error_response( error.NOT_EXISTING_ISRCS_IN_ACTIVE, list(missing_isrcs)) return response.Response() def unlock_territories( correlation_id, user, isrc, territories_to_unlock, reason): """Unlock a list of territories for single ISRC Args: correlation_id (str): The correlation id for logging the microservice instance user (str): The user id for audit table logging isrc (str): international standard recording codes territories_to_unlock (list): list of ISO-3166-1 or fully-qualified territories reason (str): lock reason Returns: response.Response """ validate_response, active_isrcs = _validate_isrcs_and_territories( [isrc], territories_to_unlock ) if not validate_response: return validate_response else: territories_to_unlock = validate_response.message unlock_isrc(isrc, territories_to_unlock, correlation_id, user) import_response = { 'isrc': isrc, 'territories': territories_to_unlock } # Append lock history history_response = ownership.get_lock_history_for_single_reason( reason, isrc) if not history_response: return history_response import_response.update(history_response.message) return response.Response(import_response) def unlock_isrc(isrc, territories_to_unlock, correlation_id, user): """Function performs unlock for single ISRC It updates active table and audit table Args: isrc (str): international standard recording code territories_to_unlock (list): list of territories to unlock correlation_id (str): correlation id for logging user (str): The user id for audit table logging """ unlock_update_audit_table( isrc, territories_to_unlock, correlation_id, user) ownership.update_active_record(isrc, {'to_unlock': territories_to_unlock}) def bulk_unlock_territories( correlation_id, user, isrcs, territories_to_unlock): """Unlock a list of ISRC/territories Args: correlation_id (str): The correlation id for logging the microservice instance user (str): The user id for audit table logging isrcs (list): list of known ISRCs territories_to_unlock (list): list of ISO-3166-1 or fully-qualified territories Returns: response.Response """ # This local import is to avoid circular references in celery tasks from masters_registry.tasks import bulk as bulk_celery_tasks validate_response, active_isrcs = _validate_isrcs_and_territories( isrcs, territories_to_unlock ) if not validate_response: return validate_response else: territories_to_unlock = validate_response.message task_id = logic_bulk_tasks.create_task( correlation_id, user, bulk_tasks_const.BULK_UNLOCK, len(isrcs), isrcs) unlock_tasks = [] for isrc in isrcs: unlock_tasks.append( bulk_celery_tasks.unlock_isrc.s( isrc, territories_to_unlock, correlation_id, user) ) chord( unlock_tasks, bulk_celery_tasks.update_bulk_unlock_status.s( task_id, len(unlock_tasks) ) ).delay() import_response = { 'isrcs': isrcs, 'territories': territories_to_unlock } return response.Response(import_response) def unlock_update_audit_table( isrc, territories_to_unlock, correlation_id, user): """Helper function for updating audit table Args: isrc (string): ISRC territories_to_unlock (list): list of ISO-3166-1 territories correlation_id (str): correlation id for logging user (str): user id for audit table logging """ records = _create_records( {isrc: territories_to_unlock}, correlation_id, user, opcode_const.UNLOCK ) ownership.update_multiple_ownerships(records)