"""Data processing logic for the Smithsonian project."""
from collections import defaultdict
from sys import exit as sys_exit

import pandas as pd

from connectors.logging import log
from constants.participant_fields import PARTICIPANT_ROLES
from constants.localization_definitions import (
    ITUNES_LANGUAGES,
    PRODUCT_LOCALIZATIONS_FIELD,
    LOCALIZATIONS_RELEASE_FILTER,
    LOCALIZATIONS_RELEASE_REQUIRED_FIELDS,
    LOCALIZATIONS_RELEASE_SHEET,
    LOCALIZATIONS_TRACK_FILTER,
    LOCALIZATIONS_TRACK_REQUIRED_FIELDS,
    LOCALIZATIONS_TRACK_SHEET,
    LOCALIZED_INPUT_LANGUAGE_FIELD,
    LOCALIZED_INPUT_RELEASE_NAME_FIELD,
    LOCALIZED_INPUT_RELEASE_VERSION_FIELD,
    LOCALIZED_INPUT_TRACK_NAME_FIELD,
    LOCALIZED_INPUT_TRACK_VERSION_FIELD,
    LOCALIZED_RELEASE_VERSION_FIELD,
    LOCALIZED_TRACK_VERSION_FIELD,
    TRACK_LOCALIZATIONS_FIELD,
)
from constants.participant_map import (
    PARTICIPANT_ROLE_MAP,
)
from constants.smithsonian_definitions import (
    PARTICIPANT_FILTER,
    SMITHSONIAN_TRACK_SHEET,
    SMITHSONIAN_PERFORMER_CONTRIBUTOR_SHEET,
)
from utils.file_utils import (
    find_invalid_cells,
    limit_rows_by_key,
    preprocess_bad_rows,
)
from utils.general_utils import (
    log_runtime,
)
from utils.pandas_converters import (
    CONVERTER_LOCALIZATIONS_TRACK,
    CONVERTER_LOCALIZATIONS_RELEASE,
    CONVERTER_TRACKS,
    generate_performers_converters,
)

from config import (
    ABORT_ON_FAIL,
    INPUT_FILE_STARTING_ROW_OFFSET,
    NROWS,
    PERFORMER_COUNT,
)


def get_participants_from_datafile(
        participants: dict, vendor_id: int = None) -> dict:
    """Collect all participants from the input data. This is a helper function
    to parse the participants data from the input file, and ease later GraphQL
    calls.

    Args:
        participants (dict): The participants data.
        vendor_id (int): (Optional) A vendor ID to hard-code. If not provided,
            the vendor ID from the participants data will be used.

    Returns:
        dict: A dict of lsits of artist names keyed by vendor ID.
    """
    participant_count = len(participants)

    log.info(f'Collecting participants for {participant_count} rows.')

    all_artists_with_vendor = []

    for row in participants:
        vendor_id = vendor_id or row['Vendor ID']
        subaccount_id = row.get("Subaccount ID") or 0

        # Participations AKA Track Artist / Featuring / etc.
        for role_name in PARTICIPANT_ROLES.keys():
            # break up all pipe delimiters
            # # ..., and remove extraneous spaces
            artists = [artist.strip() for artist in row[role_name].split('|')]

            # Update artists with vendor ID and subaccount ID
            artists = [
                (vendor_id, artist, subaccount_id)  # , role_name)
                for artist in artists if artist
            ]

            # Add all artists to the list
            all_artists_with_vendor.extend(artists)

    # Dedupe all artists at the end to save cycles
    all_artists_with_vendor = list(set(all_artists_with_vendor))

    # Make the deduped list of tuples into a list of dicts
    all_artists_with_vendor = [
        {
            'name': artist_name,
            'vendor_id': vendor_id,
            'subaccount_id': subaccount_id,
            # 'role_name': role_name
        } for vendor_id, artist_name, subaccount_id  # , role_name
        in all_artists_with_vendor
    ]

    return all_artists_with_vendor


def process_track_updates(
        track_localization_input_data: list,
        products_by_vendor_subaccount_upc: dict) -> tuple:
    """Process track updates.

    Regarding the output of this function, the `track_updates_by_vend_sub_isrc`
    dict is the most important. It contains the track updates that will be
    sent to the Orchard. The `input_tracks_by_vendor_subaccount_upc` dict is
    used to verify that all tracks from the input file are found in the graph
    response.

    The `track_updates_by_vend_sub_isrc` dict contains the input data from
    `track_localization_input_data` pivoted to a (1)dict of (2)dicts of
    (3)dicts of (4)lists keyed by vendor ID, subaccount ID, and UPC, cascading,
    with each terminal UPC key paired with a list of dicts of track data. The
    second dict contains the track updates as a (1)dict of (2)dicts of (3)dicts
    keyed by vendor ID, subaccount ID, and ISRC, with each terminal ISRC key
    paired with a dict of track updates.

    Args:
        track_localization_input_data (list): The track localization data.
        products_by_vendor_subaccount_upc (dict): The graph products by
            UPC.

    Returns:
        tuple (dict, dict): The input tracks by vendor ID, subaccount ID, and
            UPC, and the track updates by vendor ID, subaccount ID, and ISRC.
    """
    # Pivot track_localizations by UPC
    log.info('Processing Track Updates...')
    input_tracks_by_vendor_subaccount_upc = defaultdict(
        lambda: defaultdict(lambda: defaultdict(list))
    )

    for localization in track_localization_input_data:
        vend_id = localization['Vendor ID']
        sub_id = localization.get('Subaccount ID') or 0
        upc = localization['UPC']

        input_tracks_by_vendor_subaccount_upc[vend_id][sub_id][upc]\
            .append(localization)

    # Compare and report on the number of upcs from the input file
    # and the number of upcs in the graph response
    log.info('Ensuring all UPCs are found in the graph response...')
    verify_products(
        input_tracks_by_vendor_subaccount_upc,
        products_by_vendor_subaccount_upc
    )

    # Make sure there is no discrepancy between the number of tracks in the
    # track_localizations and the number of tracks in collected graph responses
    log.info('Ensuring all tracks are found in the graph response...')
    verify_tracks(
        track_localization_input_data,
        products_by_vendor_subaccount_upc)

    # Assemble the track localization payloads
    log.info('Collecting Track Updates...')
    track_updates_by_vend_sub_isrc = collect_track_updates(
        input_tracks_by_vendor_subaccount_upc,
        products_by_vendor_subaccount_upc
    )

    log.info('Adding TUIDs to Track Updates...')
    # Hydrate in the tuid for each upc:isrc pair
    track_updates_by_vend_sub_isrc = enrich_track_updates_with_tuid(
        track_updates_by_vend_sub_isrc,
        products_by_vendor_subaccount_upc
    )

    return input_tracks_by_vendor_subaccount_upc, \
        track_updates_by_vend_sub_isrc


def process_release_updates(
        release_localization_data: list,
        products_by_vendor_subaccount_upc: dict) -> tuple:
    """Process release updates.

    Args:
        release_localization_data (list): The release localization data.
        products_by_vendor_subaccount_upc (dict): The graph products by UPC.

    Returns:
        tuple (dict, dict): The release updates and the errors.
    """
    # Pivot release_localizations by UPC
    log.info('Processing Release Updates...')
    input_releases_by_vendor_subaccount_upc = defaultdict(
        lambda: defaultdict(lambda: defaultdict(list))
    )

    for localization in release_localization_data:
        vend_id = localization['Vendor ID']
        sub_id = localization.get('Subaccount ID') or 0
        upc = localization['UPC']

        input_releases_by_vendor_subaccount_upc[vend_id][sub_id][upc]\
            .append(localization)

    # Compare and report on the number of upcs from the input file
    # and the number of upcs in the graph response
    log.info('Ensuring all UPCs are found in the graph response...')
    verify_products(
        input_releases_by_vendor_subaccount_upc,
        products_by_vendor_subaccount_upc
    )

    log.info('Collecting Release Updates...')
    release_updates_by_vend_sub_upc = collect_release_updates(
        input_releases_by_vendor_subaccount_upc,
        products_by_vendor_subaccount_upc
    )

    log.info('Adding product_id to Release Updates...')
    # Hydrate in the product_id for each upc
    release_updates_by_vend_sub_upc = enrich_release_updates_with_product_id(
        release_updates_by_vend_sub_upc,
        products_by_vendor_subaccount_upc
    )

    return input_releases_by_vendor_subaccount_upc, \
        release_updates_by_vend_sub_upc


def verify_tracks(
    track_localization_input_data: list,
    products_by_vendor_subaccount_upc: dict
):
    """Verify that all tracks are found in the graph response.

    Args:
        track_localization_input_data (list): The track localization data.
        products_by_vendor_subaccount_upc (dict): The graph products by UPC.
    """
    for row in track_localization_input_data:
        isrc = row['ISRC']
        upc = row['UPC']
        vend_id = row['Vendor ID']
        sub_id = row.get('Subaccount ID') or 0

        # Find the a track in the graph response that matches the ISRC, upc,
        # vendor ID, and subaccount ID
        matching_tracks_on_upc = [
            track for track in
            products_by_vendor_subaccount_upc[vend_id][sub_id][upc]['tracks']
            if track['isrc'] == isrc
        ]

        if not matching_tracks_on_upc:
            msg = f'No matching track found in the Orchard for ISRC {isrc} ' \
                f'and UPC {upc}'
            msg = f'{msg}  on Vendor {vend_id} Subaccount {sub_id}.'
            log.error(msg)

            if ABORT_ON_FAIL:
                raise ValueError(msg)
            continue


def verify_products(
        input_tracks_by_vendor_subaccount_upc: dict,
        products_by_vendor_subaccount: dict):
    """Verify that all UPCs are found in the graph response.

    Args:
        input_tracks_by_vendor_subaccount_upc (dict): The input tracks by
            vendor ID, subaccount ID, and UPC.
        products_by_vendor_subaccount (dict): The graph products by UPC.
    """
    # Compare and report on the number of upcs from the input file
    # and the number of upcs in the graph response
    log.info('Ensuring all UPCs are found in the graph response...')
    for vend_id, subaccounts in input_tracks_by_vendor_subaccount_upc.items():
        for sub_id, upc_list in subaccounts.items():
            for upc in upc_list:
                if upc not in products_by_vendor_subaccount[vend_id][sub_id]:
                    msg = 'No matching product found in the Orchard for ' \
                        f'UPC {upc}.'

                    if ABORT_ON_FAIL:
                        raise ValueError(msg)

                    log.warning(msg)


def get_track_participants_from_graph(product: dict) -> dict:
    """Collect the current participants for the track.

    Args:
        product (dict): The product data.

    Returns:
        dict: The current participants for the track.
    """
    upc = product['upc']
    vend_id = product['label']['id']['vendorId']
    sub_id = product['label']['id']['subaccountId']

    # Collect the current participants for the track
    msg = 'Preserving Existing Track Participations'
    msg = f'{msg} for UPC {upc} on Vendor {vend_id}'
    msg = f'{msg} Subaccount {sub_id}...'
    log.debug(msg)
    curr_track_participants = {
        track['isrc']: track['participations']
        for track
        in product.get('tracks', [])
    }

    # Massage the participant data into the shape we need
    msg = 'Shaping participation payloads'
    msg = f'{msg} for UPC {upc} on Vendor {vend_id}'
    msg = f'{msg} Subaccount {sub_id}...'
    log.debug(msg)
    try:
        # This is the payload shape.
        track_participants = {
            isrc: [  # noqa
                {
                    'labelParticipantUuid':
                        p['participant']['id'],
                    'role':  # e.g. 'PERFORMER'
                        PARTICIPANT_ROLE_MAP[
                            p['participatedAs']
                        ],
                } for p in parties
            ]
            for isrc, parties in curr_track_participants.items()
        }
    except KeyError as e:
        msg = f'Error processing participant data for ISRC on UPC {upc}: {e}' \
            f' {msg} Update the `PARTICIPANT_ROLE_MAP` to include missing ' \
            'roles.'

        if ABORT_ON_FAIL:
            raise ValueError(msg)

        log.error(msg)

    return track_participants


def get_localization_updates(data, itunes_languages=None, is_track=True):
    """Collect the new localizations for each track or release.

    Args:
        data (list): The track or release data.
        itunes_languages (list): The iTunes languages.
        is_track (bool): Whether the data is for tracks or releases.

    Returns:
        dict: The updates as a dict of lists of dicts keyed by ISRC or UPC,
            with each key paired with a list of dicts of track or release
            updates.
    """
    if not itunes_languages:
        itunes_languages = ITUNES_LANGUAGES

    localization_updates = defaultdict(list)

    # TODO: ITUNES_LANGUAGES SHOULD BE AN EARLIER CALL TO SNOWFLAKE
    #   NOT A CONSTANT, AS BELOW
    for localization in data:
        if is_track:
            isrc = localization['ISRC']

        upc = localization['UPC']
        vend_id = localization['Vendor ID']
        sub_id = localization.get('Subaccount ID') or 0

        # This is the payload shape.
        localization_data = {
            'languageId': [
                i for i in itunes_languages
                if i['language'] ==
                localization[LOCALIZED_INPUT_LANGUAGE_FIELD]
            ][0]['language_id'],
        }

        if is_track:
            localization_data['trackName'] = \
                localization[LOCALIZED_INPUT_TRACK_NAME_FIELD]
            # Only add the version if it exists
            if localization.get(LOCALIZED_INPUT_TRACK_VERSION_FIELD):
                localization_data[LOCALIZED_TRACK_VERSION_FIELD] = \
                    localization.get(LOCALIZED_INPUT_TRACK_VERSION_FIELD)
        else:
            localization_data['productName'] = \
                localization[LOCALIZED_INPUT_RELEASE_NAME_FIELD]
            # Only add the version if it exists
            if localization.get(LOCALIZED_INPUT_RELEASE_VERSION_FIELD):
                localization_data[LOCALIZED_RELEASE_VERSION_FIELD] = \
                    localization.get(LOCALIZED_INPUT_RELEASE_VERSION_FIELD)

        # Add the localization to the updates
        if is_track:
            msg = f'Adding Localization to Track Updates for {isrc} '
        else:
            msg = 'Adding Localization to Product Updates '
        msg = f'{msg} for UPC {upc} on Vendor {vend_id} '
        msg = sub_id and f'{msg} Subaccount {sub_id}...' or msg
        log.debug(msg)

        if is_track:
            localization_updates[isrc].append(localization_data)
        else:
            localization_updates[upc].append(localization_data)

    return localization_updates


def combine_track_localizations(
        localization_updates: dict, track_updates: dict,
        curr_track_locales: dict) -> dict:
    """Combine the current localizations with the updates.

    Args:
        localization_updates (dict): The new localizations.
        track_updates (dict): The track updates.
        curr_track_locales (dict): The current localizations.

    Returns:
        dict: The track updates.
    """
    for isrc, update in localization_updates.items():
        # Add the track to the track updates
        if not track_updates.get(isrc):
            track_updates[isrc] = {}

        # Add the current localizations to the track updates
        track_updates[isrc][TRACK_LOCALIZATIONS_FIELD] = \
            curr_track_locales.get(isrc) or []

        # Add the new localizations to the track updates
        for new_locale in update:
            # if the new_locale is an exact match, don't add it
            if new_locale in track_updates[isrc][TRACK_LOCALIZATIONS_FIELD]:
                continue

            # if the new_locale language is not in the current locales, add it
            if not any(
                    locale['languageId'] == new_locale['languageId']
                    for locale
                    in track_updates[isrc][TRACK_LOCALIZATIONS_FIELD]):
                track_updates[isrc][TRACK_LOCALIZATIONS_FIELD] \
                    .append(new_locale)
                continue

            # If the new_locale language is in the current locales, but the
            # version is different, add it
            matching_locale = [
                locale for locale
                in track_updates[isrc][TRACK_LOCALIZATIONS_FIELD]
                if locale['languageId'] == new_locale['languageId']
            ][0]

            # Remove the matching locale and add the new locale
            if matching_locale.get(LOCALIZED_TRACK_VERSION_FIELD) != \
                    new_locale.get(LOCALIZED_TRACK_VERSION_FIELD):
                track_updates[isrc][TRACK_LOCALIZATIONS_FIELD] \
                    .remove(matching_locale)
                track_updates[isrc][TRACK_LOCALIZATIONS_FIELD] \
                    .append(new_locale)

    return track_updates


def combine_release_localizations(
        localization_updates: dict, release_updates: dict, upc: str,
        curr_release_locales: dict) -> dict:
    """Combine the current localizations with the updates.

    Args:
        localization_updates (dict): The new localizations.
        release_updates (dict): The release updates.
        upc (str): The UPC.
        curr_release_locales (dict): The current localizations.

    Returns:
        dict: The release updates.
    """
    for upc, update in localization_updates.items():
        if not release_updates.get(upc):
            release_updates[upc] = {}

        # Add the current localizations to the release updates
        release_updates[upc][PRODUCT_LOCALIZATIONS_FIELD] = \
            curr_release_locales or []

        # Add the new localizations to the release_updates
        for new_locale in update:
            # if the new_locale is an exact match, don't add it
            if new_locale in release_updates[upc][PRODUCT_LOCALIZATIONS_FIELD]:
                continue

            # if the new_locale language is not in the current locales, add it
            if not any(
                    locale['languageId'] == new_locale['languageId']
                    for locale
                    in release_updates[upc][PRODUCT_LOCALIZATIONS_FIELD]):
                release_updates[upc][PRODUCT_LOCALIZATIONS_FIELD]\
                    .append(new_locale)
                continue

            # If the new_locale language is in the current locales, but the
            # version is different, add it
            matching_locale = [
                locale for locale
                in release_updates[upc][PRODUCT_LOCALIZATIONS_FIELD]
                if locale['languageId'] == new_locale['languageId']
            ][0]

            # Remove the matching locale and add the new locale
            if matching_locale.get(LOCALIZED_RELEASE_VERSION_FIELD) != \
                    new_locale.get(LOCALIZED_RELEASE_VERSION_FIELD):
                release_updates[upc][PRODUCT_LOCALIZATIONS_FIELD] \
                    .remove(matching_locale)
                release_updates[upc][PRODUCT_LOCALIZATIONS_FIELD] \
                    .append(new_locale)

    return release_updates


def enrich_track_updates_with_tuid(
        track_updates_with_isrc: dict,
        products_by_vendor_subaccount_upc) -> dict:
    """Add the tuid to each track update.

    Args:
        track_updates_by_vend_sub_isrc (dict): The track updates.

    Returns:
        dict: The track updates.
    """
    return {
        vend_id: {
            sub_id: {
                upc: {
                    isrc: [
                        {
                            'tuid': t['tuid'] if 'tuid' in t else None,
                            **updates
                        }
                        for t in products_by_vendor_subaccount_upc
                        [vend_id][sub_id][upc]['tracks']
                        if t['isrc'] == isrc
                    ][0]  # There should only be one match
                    for isrc, updates in release.items()
                }
                for upc, release in subaccount.items()
            }
            for sub_id, subaccount in vend.items()
        }
        for vend_id, vend in track_updates_with_isrc.items()
    }


def enrich_release_updates_with_product_id(
        release_updates_by_vend_sub_upc: dict,
        products_by_vendor_subaccount_upc: dict) -> dict:
    """Add the product_id to each release update.

    Args:
        release_updates_by_vend_sub_upc (dict): The release updates.
        products_by_vendor_subaccount_upc (dict): The graph products by UPC.

    Returns:
        dict: The release updates.
    """
    return {
        vend_id: {
            sub_id: {
                upc: {
                    'product_id': products_by_vendor_subaccount_upc
                    [vend_id][sub_id][upc]['productId'],
                    **release
                }
                for upc, release in subaccount.items()
            }
            for sub_id, subaccount in vend.items()
        }
        for vend_id, vend in release_updates_by_vend_sub_upc.items()
    }


def attach_participants(
        track_updates: dict, vendor_participants: dict) -> dict:
    """Attach participant data to the track updates.

    Args:
        track_updates (dict): The track updates.
        vendor_participants (dict): The vendor participants.

    Returns:
        dict: The track updates.
    """
    for isrc, participants in vendor_participants.items():
        # Rename long var for legibility
        check_track = track_updates.get(isrc)
        if not check_track:
            msg = f'No localization updates found for ISRC {isrc}. ' \
                'No participants will be added.'

            if ABORT_ON_FAIL:
                raise ValueError(msg)

            log.warning(msg)
            continue

        track_updates[isrc]['participations'] = participants

    return track_updates


def get_current_release_localizations(product: dict) -> dict:
    """Collect the current localizations for the release.

    Args:
        product (dict): The product data.

    Returns:
        dict: The current localizations for the release.
    """
    upc = product['upc']
    vend_id = product['label']['id']['vendorId']
    sub_id = product['label']['id']['subaccountId']

    # Collect the current localizations for the release
    msg = 'Preserving Existing Release Localizations'
    msg = f'{msg} for UPC {upc} on Vendor {vend_id}'
    msg = f'{msg} Subaccount {sub_id}...'
    log.debug(msg)
    curr_release_locales = product.get(PRODUCT_LOCALIZATIONS_FIELD, [])

    # reformat the localizations to match the input data
    formatted_release_localizations = []
    for locale in curr_release_locales:
        curr_locale = {
            'languageId': locale['iTunesLanguage']['id'],
            'productName': locale['productName'],
        }

        # Only add the version if it exists
        if locale.get(LOCALIZED_RELEASE_VERSION_FIELD):
            curr_locale[LOCALIZED_RELEASE_VERSION_FIELD] = \
                locale[LOCALIZED_RELEASE_VERSION_FIELD]

        formatted_release_localizations.append(curr_locale)

    return formatted_release_localizations


def get_current_track_localizations(product: dict) -> dict:
    """Collect the current localizations for the track.

    Args:
        product (dict): The product data.

    Returns:
        dict: The current localizations for the track.
    """
    upc = product['upc']
    vend_id = product['label']['id']['vendorId']
    sub_id = product['label']['id']['subaccountId']

    # Collect the current localizations for the track
    msg = 'Preserving Existing Track Localizations'
    msg = f'{msg} for UPC {upc} on Vendor {vend_id}'
    msg = f'{msg} Subaccount {sub_id}...'
    log.debug(msg)
    curr_track_locales = {
        track['isrc']: track[TRACK_LOCALIZATIONS_FIELD]
        for track
        in product.get('tracks', [])
    }

    # reformat the localizations to match the input data
    formatted_track_localizations = defaultdict(list)
    for isrc, locales in curr_track_locales.items():
        for locale in locales:
            curr_locale = {
                'languageId': locale['iTunesLanguage']['id'],
                'trackName': locale['trackName'],
            }

            # Only add the version if it exists
            if locale.get(LOCALIZED_TRACK_VERSION_FIELD):
                curr_locale[LOCALIZED_TRACK_VERSION_FIELD] = \
                    locale[LOCALIZED_TRACK_VERSION_FIELD]

            formatted_track_localizations[isrc].append(curr_locale)

    return formatted_track_localizations


def collect_track_updates(
        input_tracks_by_vendor_subaccount_upc: dict,
        products_by_vendor_subaccount_upc: dict) -> dict:
    """Collect the track updates.

    Args:
        input_tracks_by_vendor_subaccount_upc (dict): The input tracks by
            vendor ID, subaccount ID, and UPC.
        products_by_vendor_subaccount_upc (dict): The graph products by UPC.

    Returns:
        dict: The track updates.
    """
    # Initialize defaultdicts for nested dictionaries
    # The new localizations
    locale_updates_by_vend_sub_upc_isrc = \
        defaultdict(lambda: defaultdict(lambda: defaultdict(dict)))
    # The existing participants
    participants_by_vend_sub_isrc = \
        defaultdict(lambda: defaultdict(lambda: defaultdict(dict)))
    # The completed track updates
    track_updates_by_vend_sub_isrc = \
        defaultdict(lambda: defaultdict(lambda: defaultdict(dict)))

    # We start by going release by release through the input file
    for vend_id, subaccounts in input_tracks_by_vendor_subaccount_upc.items():
        for sub_id, upc_list in subaccounts.items():
            # Build all track payloads, per release
            for upc, tracks in upc_list.items():
                # Collect the new localizations for each track
                locale_updates_by_vend_sub_upc_isrc[vend_id][sub_id][upc]\
                    .update(get_localization_updates(tracks, is_track=True))

                # Get the product for the current UPC
                product = \
                    products_by_vendor_subaccount_upc[vend_id][sub_id].get(upc)

                if not product:
                    msg = 'No matching product found in the Orchard for ' \
                        f'UPC {upc}.'

                    if ABORT_ON_FAIL:
                        raise ValueError(msg)

                    log.warning(msg)
                    continue

                # Collect the current localizations for the track
                curr_track_locales = get_current_track_localizations(product)

                # Collect the current participants for the track
                participants_by_vend_sub_isrc[vend_id][sub_id][upc].update(
                    get_track_participants_from_graph(product)
                )

                # Combine the current localizations with the updates
                track_updates_by_vend_sub_isrc[vend_id][sub_id][upc].update(
                    combine_track_localizations(
                        locale_updates_by_vend_sub_upc_isrc[vend_id][sub_id][upc],  # noqa
                        track_updates_by_vend_sub_isrc[vend_id][sub_id][upc],
                        curr_track_locales
                    )
                )

                # Rename long var for legibility
                vendor_participants = \
                    participants_by_vend_sub_isrc[vend_id][sub_id][upc]

                # Attach participant data to the track updates
                track_updates_by_vend_sub_isrc[vend_id][sub_id][upc].update(
                    attach_participants(
                        track_updates_by_vend_sub_isrc[vend_id][sub_id][upc],
                        vendor_participants
                    )
                )

    return track_updates_by_vend_sub_isrc


def collect_release_updates(
        input_releases_by_vendor_subaccount_upc: dict,
        products_by_vendor_subaccount_upc: dict) -> dict:
    """Collect the release updates.

    Args:
        input_releases_by_vendor_subaccount_upc (dict): The input releases by
            vendor ID, subaccount ID, and UPC.
        products_by_vendor_subaccount_upc (dict): The graph products by UPC.

    Returns:
        dict: The release updates.
    """
    # Initialize defaultdicts for nested dictionaries
    # The new localizations
    locale_updates_by_vend_sub_upc = \
        defaultdict(lambda: defaultdict(lambda: defaultdict(dict)))
    # The completed release updates
    release_updates_by_vend_sub_upc = \
        defaultdict(lambda: defaultdict(lambda: defaultdict(dict)))

    # We start by going release by release through the input file
    for vend_id, subaccounts in input_releases_by_vendor_subaccount_upc.items():  # noqa
        for sub_id, upc_list in subaccounts.items():
            # Build all release payloads, per release
            for upc, releases in upc_list.items():
                # Collect the new localizations for each release
                locale_updates_by_vend_sub_upc[vend_id][sub_id][upc]\
                    .update(get_localization_updates(releases, is_track=False))

                # Get the product for the current UPC
                product = \
                    products_by_vendor_subaccount_upc[vend_id][sub_id].get(upc)

                if not product:
                    msg = 'No matching product found in the Orchard for ' \
                        f'UPC {upc}.'

                    if ABORT_ON_FAIL:
                        raise ValueError(msg)

                    log.warning(msg)
                    continue

                # Collect the current localizations for the release
                curr_release_locales = \
                    get_current_release_localizations(product)

                # Combine the current localizations with the updates
                release_updates_by_vend_sub_upc[vend_id][sub_id][upc].update(
                    combine_release_localizations(
                        locale_updates_by_vend_sub_upc[vend_id][sub_id][upc],
                        release_updates_by_vend_sub_upc[vend_id][sub_id][upc],
                        upc, curr_release_locales
                    )
                )

    return release_updates_by_vend_sub_upc


def pivot_list_of_graph_products(graph_product_list: list) -> dict:
    """Pivot a list of graph products by vendor ID, subaccount ID, and UPC.

    Args:
        graph_product_list (list): A list of graph `productByUpc` responses as
            dicts.

    Returns:
        dict: The graph products by vendor ID, subaccount ID, and UPC.
    """
    return {
        vendor_id: {
            subaccount_id: {
                product['upc']: product
                for product in graph_product_list
                if product['label']['id']['vendorId'] == vendor_id
                and product['label']['id']['subaccountId'] == subaccount_id
            }
            for subaccount_id in set([
                product['label']['id']['subaccountId']
                for product in graph_product_list
                if product['label']['id']['vendorId'] == vendor_id
            ])
        }
        for vendor_id in set([
            product['label']['id']['vendorId']
            for product in graph_product_list
        ])
    }


@log_runtime
def preprocess_smithsonian_contributor_data(filename: str = None):
    """Process Smithsonian input files to create contributor update payload.

    Args:
        filename (str, optional): The filename to process. Defaults to None.

    Returns:
        tuple (dict, dict): The preprocessed Smithsonian data.
    """
    log.info(f'Preprocessing Smithsonian Data from {filename}....')

    if INPUT_FILE_STARTING_ROW_OFFSET:
        log.info(f'Skipping {INPUT_FILE_STARTING_ROW_OFFSET} rows.')

        def skip_func(x):
            return x in range(1, INPUT_FILE_STARTING_ROW_OFFSET + 1)
    else:
        skip_func = None

    # If NROWS is set, limit the number of items
    num_rows = NROWS
    if NROWS > 0:
        log.info(f'Limiting data to {NROWS} rows.')
    # Ints come in, negatives are None
    elif NROWS <= 0:
        num_rows = None

    # Get participant data
    participations = pd.read_excel(
        filename,
        sheet_name=SMITHSONIAN_TRACK_SHEET,
        nrows=num_rows,
        skiprows=skip_func,
        usecols=PARTICIPANT_FILTER,
        converters=CONVERTER_TRACKS,
        header=0
    )

    participations.fillna('', inplace=True)

    # Convert the entire DataFrame to a dictionary
    participations = participations.to_dict(orient='records')

    # Process limiting config vars
    participations = limit_rows_by_key(participations)

    # TODO: Add preprocess_bad_rows for participations
    #
    # Create SMITHSONIAN_TRACK_REQUIRED_FIELDS
    # participations = preprocess_bad_rows(
    #     participations, SMITHSONIAN_TRACK_REQUIRED_FIELDS)

    # Generate converters for contributors
    CONVERTER_CONTRIBS = generate_performers_converters(PERFORMER_COUNT)

    # contributor requires looping thru all columns, then lookup RoleID & Role
    # Type
    performers = pd.read_excel(
        filename,
        sheet_name=SMITHSONIAN_PERFORMER_CONTRIBUTOR_SHEET,
        nrows=num_rows,
        skiprows=skip_func,
        converters=CONVERTER_CONTRIBS,
        header=0
    )

    # Risk casting values to incorrect types?
    performers.fillna('', inplace=True)

    # Convert the entire DataFrame to a dictionary
    performers = performers.to_dict(orient='records')

    # Process limiting config vars
    performers = limit_rows_by_key(performers)

    # TODO: Add preprocess_bad_rows for performers
    #
    # Create SMITHSONIAN_PERFORMER_CONTRIBUTOR_REQUIRED_FIELDS
    # performers = preprocess_bad_rows(
    #     performers, SMITHSONIAN_PERFORMER_CONTRIBUTOR_REQUIRED_FIELDS)

    log.info('Preprocessing Smithsonian Complete.')

    return participations, performers


@log_runtime
def preprocess_localization_data(filename: str = None):
    """Preprocess a file for localizations.

    Args:
        filename (str, optional): The filename to process. Defaults to None.

    Returns:
        tuple (dict, dict): The preprocessed Smithsonian data.
    """
    if INPUT_FILE_STARTING_ROW_OFFSET:
        log.info(f'Skipping {INPUT_FILE_STARTING_ROW_OFFSET} rows.')

        def skip_func(x):
            return x in range(1, INPUT_FILE_STARTING_ROW_OFFSET + 1)
    else:
        skip_func = None

    # If NROWS is set, limit the number of items
    num_rows = NROWS
    if NROWS > 0:
        log.info(f'Limiting data to {NROWS} rows.')
    # Ints come in, negatives are None
    elif NROWS <= 0:
        num_rows = None

    # -- Calculations in this section can be done in pandas.... --------------
    # Read Track-level data
    try:
        track_localizations = pd.read_excel(
            filename,
            sheet_name=LOCALIZATIONS_TRACK_SHEET,
            nrows=num_rows,
            skiprows=skip_func,
            usecols=LOCALIZATIONS_TRACK_FILTER,
            converters=CONVERTER_LOCALIZATIONS_TRACK,
            header=0
        )
    except ValueError as e:
        log.error(f"Error reading Excel file: {e}")
        log.error(
            "Please check the data in the 'Track Name Localizations' "
            "sheet for invalid values.")
        log.error(
            'Attempting to identify invalid cells in the Track Localizations '
            'sheet...')
        try:
            # Read the data without converters to find the invalid cells
            non_conv_track_localizations = pd.read_excel(
                filename,
                sheet_name=LOCALIZATIONS_TRACK_SHEET,
                nrows=num_rows,
                skiprows=skip_func,
                usecols=LOCALIZATIONS_TRACK_FILTER,
                header=0
            )
            invalid_cells = find_invalid_cells(
                non_conv_track_localizations, CONVERTER_LOCALIZATIONS_TRACK)
            for idx, col, value in invalid_cells:
                log.error(
                    f"Invalid value '{value}' found at row {idx + 1}, "
                    f"column '{col}'")

            track_localizations = pd.read_excel(
                filename,
                sheet_name=LOCALIZATIONS_TRACK_SHEET,
                nrows=num_rows,
                skiprows=skip_func,
                usecols=LOCALIZATIONS_TRACK_FILTER,
                converters=CONVERTER_LOCALIZATIONS_TRACK,
                header=0,
            )
        except Exception as e:
            log.error(
                f"An additional error occurred while trying to identify "
                f"invalid cells: {e}")
            sys_exit(1)

    # If no rows are found, log an error and exit
    if track_localizations.empty:
        log.error('No rows found in the Track Localizations sheet.')
        sys_exit(1)

    # Risk casting values to incorrect types?
    track_localizations.fillna('', inplace=True)

    # Convert the entire DataFrame to a dictionary
    track_localizations = track_localizations.to_dict(orient='records')

    # Process limiting config vars
    track_localizations = limit_rows_by_key(track_localizations)

    track_localizations = preprocess_bad_rows(
        track_localizations, LOCALIZATIONS_TRACK_REQUIRED_FIELDS)

    # Read Release-level data
    try:
        release_localizations = pd.read_excel(
            filename,
            sheet_name=LOCALIZATIONS_RELEASE_SHEET,
            nrows=num_rows,
            skiprows=skip_func,
            usecols=LOCALIZATIONS_RELEASE_FILTER,
            converters=CONVERTER_LOCALIZATIONS_RELEASE,
            header=0
        )
    except ValueError as e:
        log.error(f"Error reading Excel file: {e}")
        log.error(
            "Please check the data in the 'Release Name Localizations' "
            "sheet for invalid values.")
        log.error(
            'Attempting to identify invalid cells in the Release '
            'Localizations sheet...')
        try:
            # Read the data without converters to find the invalid cells
            non_conv_release_localizations = pd.read_excel(
                filename,
                sheet_name=LOCALIZATIONS_RELEASE_SHEET,
                nrows=num_rows,
                skiprows=skip_func,
                usecols=LOCALIZATIONS_RELEASE_FILTER,
                header=0
            )
            invalid_cells = find_invalid_cells(
                non_conv_release_localizations,
                CONVERTER_LOCALIZATIONS_RELEASE)
            for idx, col, value in invalid_cells:
                log.error(
                    f"Invalid value '{value}' found at row {idx + 1}, "
                    f"column '{col}'")
        except Exception as e:
            log.error(
                f"An additional error occurred while trying to identify "
                f"invalid cells: {e}")
            sys_exit(1)

    # Risk casting values to incorrect types?
    release_localizations.fillna('', inplace=True)

    # Convert the entire DataFrame to a dictionary
    release_localizations = release_localizations.to_dict(orient='records')

    # Process limiting config vars
    release_localizations = limit_rows_by_key(release_localizations)

    release_localizations = preprocess_bad_rows(
        release_localizations, LOCALIZATIONS_RELEASE_REQUIRED_FIELDS)

    log.info('Preprocessing Localizations Complete.')

    return track_localizations, release_localizations


def process_localization_responses(
        track_localization_data: list,
        release_localization_data: list) -> dict:
    """
    Combine the outputs of two separate processing functions into a single
    dict.

    Args:
        track_localization_data (list): The track localization data.
        release_localization_data (list): The release localization data.

    Returns:
        dict: The combined localization data.
    """
    return {
        'track': track_localization_data,
        'product': release_localization_data
    }


def process_localization_errors(
        track_localization_errors: list,
        release_localization_errors: list) -> dict:
    """
    Combine the outputs of two separate processing functions into a single
    dict.

    Args:
        track_localization_errors (list): The track localization errors.
        release_localization_errors (list): The release localization errors.

    Returns:
        dict: The combined localization errors.
    """
    return {
        'track': track_localization_errors,
        'product': release_localization_errors
    }