"""Logic layer for handling messages from art_relations db.""" from __future__ import annotations import datetime import re import uuid from typing import Dict, List import backoff import neo4j from lambdacommon.common_config import logger from pythonfeatures import pythonfeatures from pythonfeatures.constants.split import FEATURE_ENABLED from src.constants.general import ( FACEBOOK_STORE_ID, FEATURE_MOVE_PROJECT, INSTAGRAM_STORE_ID, TWITTER_STORE_ID, YOUTUBE_STORE_ID, ) from src.constants.queries import SUBACCOUNT_LABELS, VENDOR_LABELS from src.logic.art_relations.queries import move_project from src.logic.art_relations.queries import queries as ar_queries from src.logic.retry_exception import ProjectMoveError from src.model import neo4j_graph, ows_account, ows_product, ows_track, ows_users from src.model.neo4j_graph import neo4j_session def vendor_insert_update(record): """Handle insert and update to a vendor records. Args: record (dict): Representation of a vendor row. """ query = ar_queries.vendor_insert_update_cypher neo4j_graph.run(query, record['data']) def company_brand_insert_update(record): """Handle insert and update to a company_brand record. Creates or updates a CompanyBrand node in Neo4j and links it to the appropriate ParentCompany via a BELONGS_TO relationship. Args: record (dict): Representation of a company_brand row with fields: - id: The company brand ID - name: Short name (e.g., 'theorchard', 'awal') - uuid: The company brand UUID - display_name: Display name (e.g., 'The Orchard', 'AWAL') - parent_company_id: FK to parent_company table """ data = record['data'] parent_company_id = data.get('parent_company_id') # Get parent company UUID from mapping parent_company_uuid = ar_queries.PARENT_COMPANY_UUID_MAP.get(parent_company_id) if not parent_company_uuid: logger.warning( f'Unknown parent_company_id {parent_company_id} for company_brand ' f'{data.get("name")} (id={data.get("id")}). Skipping parent company link.' ) return params = { **data, 'parent_company_uuid': parent_company_uuid, } query = ar_queries.company_brand_insert_update_cypher neo4j_graph.run(query, params) def subaccount_insert_update(record): """Handle insert and update to a subaccount records. Args: record (dict): Representation of a vendor row. """ query = ar_queries.subaccount_insert_update_with_uuid_cypher neo4j_graph.run(query, record['data']) def artist_info_insert(record): """Handle inserted artist_info records. Args: record (dict): Representation of a new record. """ params = {'is_distributor': ows_account.is_distributor(record['data']['vendor_id']), **record['data']} result = neo4j_graph.run(ar_queries.artist_info_insert_cypher, params) logger.info('Setting UUIDs to related LabelParticipant...') neo4j_graph.run( ar_queries.set_label_uuids_for_label_participant_by_params, { 'name': record['data']['name'], 'vendor_id': record['data']['vendor_id'], 'subaccount_id': 0, }, ) logger.info('UUIDs were successfully set') _set_label_uuids_to_label_participant(result) def artist_info_update(record): """Handle updated artist_info records. The source database updates this data in ways that can change the meaning drastically in The Music Graph. It is easier for us to pretend that whenever we get an update we instead got a delete message for the original data and an insert for the new data. In this case the insert handle deletions from old records. Args: record (dict): Representation of a new record. """ artist_info_insert(record) def artist_info_delete(record): """Handle deleted artist_info records. Args: record (dict): Representation of a new record. """ neo4j_graph.run(ar_queries.artist_info_delete_cypher, record['data']) def update_label_profile(record): """Update vend_contact_roles records for labelprofile. Args: record (dict): Representation of a new record. """ # we need role names not the ids. So get it from ows-users. result = ows_users.get_vend_contact_user_roles(record['data']['vend_contact_id']) profile_roles = [x.lower().replace(' ', '_') for x in result['role_names']] record['data']['roles'] = profile_roles neo4j_graph.run(ar_queries.labelprofile_replace_roles_cypher, record['data']) return profile_roles def insightsprofile_add_access(record): """Add some access to Insights Profile.""" result = ows_users.get_user_minimum_details(f'alw:{record["data"]["vend_contact_id"]}') if not result.get('account'): return resource_type = VENDOR_LABELS record['data']['resource_id'] = result.get('account').get('vendor_id') if result.get('account').get('subaccount_id'): # subaccount access resource_type = SUBACCOUNT_LABELS record['data']['resource_id'] = result.get('account').get('subaccount_id') resource_uuid = str(uuid.uuid4()) if resource_type == VENDOR_LABELS: resource_uuid = '' if resource_type == SUBACCOUNT_LABELS: resource_uuid = '' record['data']['uuid'] = resource_uuid neo4j_graph.run(ar_queries.create_insightsprofile_if_not_exists_cypher, record['data']) query = ar_queries.insightsprofile_add_access_cypher.format(resource_type=resource_type) neo4j_graph.run(query, record['data']) def insightsprofile_delete_access(record): """Remove some access from Insights Profile.""" result = ows_users.get_user_minimum_details(f'alw:{record["data"]["vend_contact_id"]}') if not result.get('account'): return resource_type = VENDOR_LABELS record['data']['resource_id'] = result.get('account').get('vendor_id') if result.get('account').get('subaccount_id'): # subaccount access resource_type = SUBACCOUNT_LABELS record['data']['resource_id'] = result.get('account').get('subaccount_id') query = ar_queries.insightsprofile_delete_access_cypher.format(resource_type=resource_type) neo4j_graph.run(query, record['data']) def settingsprofile_add_access(record): """Add some access to Settings Profile.""" result = ows_users.get_user_minimum_details(f'alw:{record["data"]["vend_contact_id"]}') if not result.get('account'): return resource_type = VENDOR_LABELS record['data']['resource_id'] = result.get('account').get('vendor_id') if result.get('account').get('subaccount_id'): resource_type = SUBACCOUNT_LABELS record['data']['resource_id'] = result.get('account').get('subaccount_id') resource_uuid = str(uuid.uuid4()) if resource_type == VENDOR_LABELS: resource_uuid = '' if resource_type == SUBACCOUNT_LABELS: resource_uuid = '' record['data']['uuid'] = resource_uuid query = ar_queries.settingsprofile_add_access_cypher.format(resource_type=resource_type) neo4j_graph.run(query, record['data']) def settingsprofile_delete_access(record): """Remove some access from Settings Profile.""" result = ows_users.get_user_minimum_details(f'alw:{record["data"]["vend_contact_id"]}') if not result.get('account'): return resource_type = VENDOR_LABELS record['data']['resource_id'] = result.get('account').get('vendor_id') if result.get('account').get('subaccount_id'): # subaccount access resource_type = SUBACCOUNT_LABELS record['data']['resource_id'] = result.get('account').get('subaccount_id') query = ar_queries.settingsprofile_delete_access_cypher.format(resource_type=resource_type) neo4j_graph.run(query, record['data']) def project_insert(record): """Handle inserted project records. Args: record (dict): Representation of a new record. """ result = neo4j_graph.run(ar_queries.project_insert_cypher, record['data']) _set_label_uuids_to_label_participant(result) @neo4j_session def project_update(record, neo4j_session): """Handle update project records. Args: record (dict): Representation of a new record. neo4j_session (neo4j.Session): Neo4j Session instance. """ result = neo4j_session.run(ar_queries.project_update_cypher, record['data']) _set_label_uuids_to_label_participant(result.data()) if _project_moved(record): move_project_ff_enabled = ( pythonfeatures.get_single_feature_by_attributes(FEATURE_MOVE_PROJECT, {}).message == FEATURE_ENABLED ) logger.info(f'Project has been moved. Project id {record["data"]["project_id"]}') if move_project_ff_enabled: try: _move_project_dependencies(neo4j_session, record) logger.info('Project dependencies have been successfully moved.') except ProjectMoveError as e: logger.error(f'Failed to move project dependencies due to: {str(e)}') except Exception as e: logger.error(f'Unexpected error during the move project process due to: {str(e)}') def _project_moved(record): """Check whether the project was moved to another vendor or subaccount.""" return 'subaccount_id' in record.get('old', {}) or 'vendor_id' in record.get('old', {}) def _move_project_dependencies(neo4j_session, record): """Move project dependencies to the new subaccount.""" logger.info(f'Start moving dependencies for project id {record["data"]["project_id"]}') for step_number, query in enumerate(move_project.MOVE_PROJECT_QUERIES, start=1): logger.info(f'Running step {step_number} of {len(move_project.MOVE_PROJECT_QUERIES)}') _run_move_project_query(neo4j_session, query, record['data']) @backoff.on_exception(backoff.expo, exception=ProjectMoveError, max_tries=3) def _run_move_project_query(neo4j_session, query, record): """Run a query and raise an exception if there are errors.""" result = neo4j_session.run(query, record) errors = _extract_errors_from_move_query(result.data()) if errors: logger.info(f'Errors during the move project process: {errors}') raise ProjectMoveError(message=errors) def _extract_errors_from_move_query(query_result): """Extract errors from the query result.""" errors = [] for result in query_result: if result.get('errorMessages'): errors.append(result['errorMessages']) return errors def project_delete(record): """Handle deleted project records. Args: record (dict): Representation of a deleted record. """ neo4j_graph.run(ar_queries.project_delete_cypher, record['data']) def product_insert(record): """Handle inserted product records. Args: record (dict): Representation of a new record. """ record_data = record['data'] product = ows_product.get_product(record_data['release_id']) params = {**product, **record_data} neo4j_graph.run(ar_queries.product_insert_cypher, params) def product_delete(record): """Handle deleted product records. Args: record (dict): Representation of a deleted record. """ neo4j_graph.run(ar_queries.product_delete_cypher, record['data']) @neo4j_session def product_update(record, neo4j_session): """Handle updated product records. Args: record (dict): Representation of a new record. neo4j_session (neo4j.Session): Neo4j Session instance. """ record_data = record['data'] product = ows_product.get_product(record_data['release_id']) params = {**product, **record_data} if _product_moved(record): logger.info(f'Product has been moved. Product id {record["data"]["release_id"]}') moved_dependencies = neo4j_session.write_transaction(_move_product_dependencies, record, params=params) _set_label_uuids_to_moved_label_participants(moved_dependencies.get('label_participants')) logger.info('Product dependencies have been moved.') else: logger.info('Normal product update.') neo4j_session.run(ar_queries.product_insert_cypher, **params) def _product_moved(record): """Check whether the product was moved to another project.""" return bool(record.get('old', {}).get('project_id')) def _move_product_dependencies(tx, record, params=None): """Move product dependencies to the new project.""" moved_dependencies = {} tx.run(ar_queries.product_insert_cypher, params or record['data']) tx.run(ar_queries.product_move_remove_old_project_cypher, record['data']) if _product_moved_to_subaccount(record): tx.run(ar_queries.product_move_copy_label_participants_cypher, record['data']) tx.run(ar_queries.product_move_remove_old_label_participants, record['data']) tx.run(ar_queries.product_move_copy_track_label_participants_cypher, record['data']) tx.run(ar_queries.product_move_remove_old_track_label_participants, record['data']) tx.run(ar_queries.product_move_copy_label_sound_recordings_cypher, record['data']) label_participants = tx.run(ar_queries.moved_label_participants_cypher, record['data']) moved_dependencies['label_participants'] = [r['lp'] for r in label_participants] return moved_dependencies def _product_moved_to_subaccount(record): """Check whether the product was moved to another subaccount.""" return 'subaccount_id' in record.get('old', {}) def _set_label_uuids_to_moved_label_participants(records): """Set label UUIDs to moved LabelParticipant and link it to GP.""" if not records: return logger.info('Setting UUIDs to moved LabelParticipant nodes and link it to GP.') for record in records: neo4j_graph.run(ar_queries.set_label_uuids_for_label_participant_by_uuid, {'uuid': record['uuid']}) neo4j_graph.run(ar_queries.link_global_participant_to_label_participant, {'uuid': record['uuid']}) logger.info('UUIDs were successfully set') def release_artist_insert(record, neo4j_runner=neo4j_graph): """Handle inserted release_artist records. Args: record (dict): Representation of a new record. neo4j_runner (object): Object capable of running write Neo4j queries. """ params = _get_release_artist_insert_params(record) logger.info('New release_artist record with params: {}'.format(params)) result = neo4j_runner.run(ar_queries.release_artist_insert_cypher, params) _set_label_uuids_to_label_participant(result) @neo4j_session def release_artist_update(record, neo4j_session): """Handle updated release_artist records. The source database updates this data in ways that can change the meaning drastically in The Music Graph. It is easier for us to pretend that whenever we get an update we instead got a delete message for the original data and an insert for the new data. Args: record (dict): Representation of a new record. neo4j_session (neo4j.Session): Neo4j Session instance. """ old_data = {**record['data'], **record['old']} old_record = {'data': old_data} insert_params = _get_release_artist_insert_params(record) logger.info('New release_artist record with params: {}'.format(insert_params)) with neo4j_session.begin_transaction() as neo4j_transaction: release_artist_delete(old_record, neo4j_transaction) logger.info('release_artist record was deleted params: {}'.format(old_record)) result = neo4j_transaction.run(ar_queries.release_artist_insert_cypher, insert_params) query_result = result.single() logger.info('release_artist record was recreated params: {}'.format(record)) _set_label_uuids_to_label_participant(query_result) def _get_release_artist_insert_params(record): """Get params for release_artist insert operation.""" record_data = record['data'] product = ows_product.get_product(record_data['release_id']) logger.info('Product information for release_artist product: {}'.format(product)) return {**product, **record_data} def release_artist_delete(record, neo4j_runner=neo4j_graph): """Handle deleted release_artist records. Args: record (dict): Representation of a new record. neo4j_runner (object): Object capable of running write Neo4j queries. """ neo4j_runner.run(ar_queries.release_artist_delete_cypher, record['data']) def track_insert(record): """Handle inserted track records. Args: record (dict): Representation of a new record. """ record_data = record['data'] release_id = record_data.get('release_id') if release_id: logger.info('Adding new track record with release_id {} and record: {}'.format(release_id, record)) product = ows_product.get_product(release_id) logger.info('Product information {}'.format(product)) params = {**product, **record_data} logger.info('Creating new track with params {}'.format(params)) neo4j_graph.run(ar_queries.track_insert_with_product_cypher, params) else: logger.info('Adding new track without release_id record: {}'.format(record)) neo4j_graph.run(ar_queries.track_insert_cypher, record_data) def track_update(record): """Handle updated track records. Args: record (dict): Representation of a new record. """ record_data = record['data'] release_id = record_data.get('release_id') if release_id: logger.info('Updating record with release_id {} and record: {}'.format(release_id, record)) product = ows_product.get_product(release_id) logger.info('Product information {}'.format(product)) params = {**product, **record_data} logger.info('Updating track with params {}'.format(params)) neo4j_graph.run(ar_queries.track_update_cypher, params) else: logger.info('Updating track without release_id record: {}'.format(record)) neo4j_graph.run(ar_queries.track_insert_cypher, record_data) def track_delete(record): """Handle deleted track records. Args: record (dict): Representation of a new record. """ neo4j_graph.run(ar_queries.track_delete_cypher, record['data']) def track_artist_insert(record, neo4j_runner=neo4j_graph): """Handle inserted track_artist records. Args: record (dict): Representation of a new record. neo4j_runner (object): Object capable of running write Neo4j queries. """ track_artist_insert_params = _get_track_participant_insert_params(record) logger.info('New track_artist record with params: {}'.format(track_artist_insert_params)) result = neo4j_runner.run(ar_queries.track_artist_insert_cypher, track_artist_insert_params) _set_label_uuids_to_label_participant(result) neo4j_runner.run(ar_queries.track_set_sequence_numbers, {'track_id': record['data']['track_id']}) @neo4j_session def track_artist_update(record, neo4j_session): """Handle updated track_artist records. The source database updates this data in ways that can change the meaning drastically in The Music Graph. It is easier for us to pretend that whenever we get an update we instead got a delete message for the original data and an insert for the new data. Args: record (dict): Representation of a new record. neo4j_session (neo4j.Session): Neo4j Session instance. """ old_data = {**record['data'], **record['old']} old_record = {'data': old_data} track_artist_insert_params = _get_track_participant_insert_params(record) with neo4j_session.begin_transaction() as neo4j_transaction: track_artist_delete(old_record, neo4j_transaction) logger.info('track_artist record was deleted params: {}'.format(old_record)) result = neo4j_transaction.run(ar_queries.track_artist_insert_cypher, track_artist_insert_params) query_result = result.single() logger.info('track_artist record was recreated params: {}'.format(record)) neo4j_transaction.run(ar_queries.track_set_sequence_numbers, {'track_id': record['data']['track_id']}) _set_label_uuids_to_label_participant(query_result) def track_artist_delete(record, neo4j_runner=neo4j_graph): """Handle deleted track_artist records. Args: record (dict): Representation of a new record. neo4j_runner (object): Object capable of running write Neo4j queries. """ record_data = record['data'] neo4j_runner.run(ar_queries.track_artist_delete_cypher, record_data) neo4j_runner.run(ar_queries.track_set_sequence_numbers, {'track_id': record_data['track_id']}) def track_writer_insert(record, neo4j_runner=neo4j_graph): """Handle inserted track_writer records. Args: record (dict): Representation of a new record. neo4j_runner (object): Object capable of running write Neo4j queries. """ track_writer_insert_params = _get_track_participant_insert_params(record, 'unique_track_id') result = neo4j_runner.run(ar_queries.track_writer_insert_cypher, track_writer_insert_params) _set_label_uuids_to_label_participant(result) neo4j_runner.run(ar_queries.track_set_sequence_numbers, {'track_id': record['data']['unique_track_id']}) @neo4j_session def track_writer_update(record, neo4j_session): """Handle updated track_writer records. The source database updates this data in ways that can change the meaning drastically in The Music Graph. It is easier for us to pretend that whenever we get an update we instead got a delete message for the original data and an insert for the new data. Args: record (dict): Representation of a new record. neo4j_session (neo4j.Session): Neo4j Session instance. """ old_data = {**record['data'], **record['old']} old_record = {'data': old_data} track_writer_insert_params = _get_track_participant_insert_params(record, 'unique_track_id') with neo4j_session.begin_transaction() as neo4j_transaction: track_writer_delete(old_record, neo4j_transaction) logger.info('track_writer record was deleted params: {}'.format(old_record)) result = neo4j_transaction.run(ar_queries.track_writer_insert_cypher, track_writer_insert_params) query_result = result.single() logger.info('track_writer record was recreated params: {}'.format(record)) neo4j_transaction.run(ar_queries.track_set_sequence_numbers, {'track_id': record['data']['unique_track_id']}) _set_label_uuids_to_label_participant(query_result) def _get_track_participant_insert_params(record, track_id_column='track_id'): """Get params for track_artist and track_writer insert operations.""" record_data = record['data'] track = ows_track.get_track(record_data[track_id_column]) logger.info('Track information for track_artist track: {}'.format(track)) product = ows_product.get_product(track['product_id']) logger.info('Product information for track_artist product: {}'.format(product)) return {**product, **track, **record_data} def track_writer_delete(record, neo4j_runner=neo4j_graph): """Handle deleted track_writer records. Args: record (dict): Representation of a new record. neo4j_runner (object): Object capable of running write Neo4j queries. """ record_data = record['data'] neo4j_runner.run(ar_queries.track_writer_delete_cypher, record_data) neo4j_runner.run(ar_queries.track_set_sequence_numbers, {'track_id': record_data['unique_track_id']}) def participant_external_link_upsert(record): """Handle inserted/updated participant_external_identifier records. Args: record (dict): Representation of a new record. """ record_data = record['data'] if record_data['store_id'] == TWITTER_STORE_ID: neo4j_graph.run(ar_queries.participant_external_link_upsert_twitter_cypher, record_data) elif record_data['store_id'] == YOUTUBE_STORE_ID: neo4j_graph.run(ar_queries.participant_external_link_upsert_youtube_cypher, record_data) elif record_data['store_id'] == FACEBOOK_STORE_ID: facebook_id = record_data['store_artist_id'] pattern = 'profile.php\\?id=(?P.+)' cleaned_store_id = facebook_id if cleaned_store_id: found_pattern = re.match(pattern, cleaned_store_id) if found_pattern: cleaned_store_id = found_pattern.group('id') record_data['store_artist_id'] = cleaned_store_id neo4j_graph.run(ar_queries.participant_external_link_upsert_facebook_cypher, record_data) elif record_data['store_id'] == INSTAGRAM_STORE_ID: neo4j_graph.run(ar_queries.participant_external_link_upsert_instagram_cypher, record_data) else: raise NotImplementedError def _set_label_uuids_to_label_participant(query_result: List[Dict] | neo4j.Result) -> None: """Update LabelParticipant node with CompanyBrand, Vendor and Subaccount UUID values.""" if query_result: if isinstance(query_result, neo4j.Record): uuid = query_result['lp']['uuid'] elif isinstance(query_result, List): uuid = query_result[0]['lp']['uuid'] else: raise NotImplementedError logger.info('Setting UUIDs to related LabelParticipant...') neo4j_graph.run(ar_queries.set_label_uuids_for_label_participant_by_uuid, {'uuid': uuid}) logger.info('Link LabelParticipant to GlobalParticipant.') neo4j_graph.run(ar_queries.link_global_participant_to_label_participant, {'uuid': uuid}) logger.info('UUIDs were successfully set and linked to GP.') else: logger.info('LabelParticipant node is not found in the query result.') def vendor_service_tier_insert(record): """Add IN_SERVICE_TIER relationship between a vendor and service tier.""" neo4j_graph.run( ar_queries.add_service_tier_for_vendor_cypher, { 'vendor_id': record['data']['vendor_id'], 'service_tier_uuid': record['data']['service_tier_uuid'], # YYYY-MM-DDTHH:MM:SS.ffffff+00:00 'current_time': datetime.datetime.now(tz=datetime.timezone.utc), }, ) def vendor_service_tier_delete(record): """Update IN_SERVICE_TIER relationship to DELETED_IN_SERVICE_TIER.""" neo4j_graph.run( ar_queries.delete_service_tier_for_vendor_cypher, { 'vendor_id': record['data']['vendor_id'], 'service_tier_uuid': record['data']['service_tier_uuid'], 'current_time': datetime.datetime.now(tz=datetime.timezone.utc), }, )