"""This script will check a folder, grab it's folder-based UPC's and all
contained WAV files and compare the list to a pair of passed lists of UPC's
and tracks."""
from types import SimpleNamespace
import os
import re
from fnmatch import fnmatch

from integration_scripts import logger
from integration_scripts.file_utils import get_all_dirs_in_subdirs, \
    get_all_files_in_subdirs
from integration_scripts.general_use import dedupe_list, \
    get_duplicates_in_list, compare_lists, represents_int, check_consecutive

IGNORE_FILES = ['.DS_Store', 'desktop.ini', 'Thumbs.db', '._*']

separator_long = \
    '{} -------------------------------------------------------------------'

separator_short = '{} --------------------------------------------'


class IngestionAnalysis:

    def __init__(self):
        # each element is a dict with dir='', upc=''
        self.duplicate_upcs = dict(good=dict(), bad=dict())

        # each element is a dict with dir='', upc='', filename=''
        self.duplicate_tracks = dict()

        # All found audio files
        self.audio_file_list = list()
        self.cover_file_list = list()

        # All upc's from folders with valid structure
        self.upc_folder_dict = dict()

        # All upc-like values from folders with invalid structure
        self.bad_upc_dict = dict()

        # All folders in the root path which do not even start with numbers
        self.bad_folder_list = list()

        # All files with good extensions but bad names.
        self.bad_track_list = dict()
        self.bad_cover_list = dict()

        # All files, in upc-like folders that don't fit the structure.
        self.bad_file_list = list()

        # All otherwise good files with wrong file type extension
        self.bad_audio_extension_list = dict()
        self.bad_cover_extension_list = dict()

        # Files that fit the structure but are misplaced
        self.misplaced_audio_files = dict()
        self.misplaced_cover_files = dict()

    # def add_dupe_upc(sub, upc, file_name=None):
    #
    #     if file_name:
    #         self.duplicate_tracks[upc] =

    def validate_upc_folder(self, path, pattern=None):
        """Check if folder matches valid upc structure

        Args:
            path:

        Returns:

        """
        # Pattern for valid upc-folders
        if not pattern:
            pattern = \
                r'(?P<upc>^\d{1,13})(?P<separator>[\ \-\_]{0,3})(?P<extra>.*)'

        upc_folder = re.match(pattern, path)

        if upc_folder:
            folder_upc = upc_folder.group('upc')
            existing_upc = self.upc_folder_dict.get(folder_upc)

            if existing_upc:
                if not self.duplicate_upcs['good'].get(folder_upc):
                    self.duplicate_upcs['good'][folder_upc] = list()
                    other_path = self.upc_folder_dict[folder_upc][
                        'meta'].path
                    self.duplicate_upcs['good'][folder_upc].append(other_path)

                self.duplicate_upcs['good'][folder_upc].append(path)
            else:
                self.upc_folder_dict[folder_upc] = dict(
                    meta=SimpleNamespace(
                        upc=folder_upc,
                        separator=upc_folder.group('separator'),
                        extra=upc_folder.group('extra'),
                        path=path),
                    tracks=dict(),
                    covers=list())
            return True
        else:
            return False

    # def validate_upc_folder(self, path):
    #     # Check if folder matches valid upc structure
    #     # Pattern for valid upc-folders
    #     folder_pattern = \
    #         r'(?P<upc>\d{1,13})(?P<separator>[\ \-\_]{1,3})(?P<extra>.*)'
    #
    #     upc_folder = re.match(folder_pattern, path)
    #
    #     if upc_folder:
    #         existing_upc = self.upc_folder_dict.get(upc_folder.group('upc'))
    #         self.upc_folder_dict[upc_folder.group('upc')] = \
    #             SimpleNamespace(upc=upc_folder.group('upc'),
    #                             separator=upc_folder.group('separator'),
    #                             extra=upc_folder.group('extra'))
    #         return True
    #     else:
    #         return False

    def validate_broken_folder(self, path):
        """Check if a folder has any kind of UPC-like value in the front."""
        # Pattern for any folder beginning with a number
        loose_folder_pattern = r'(?P<upc>^\d{1,100})(?P<extra>.*)'

        upc_like_folder = re.match(loose_folder_pattern, path)

        if upc_like_folder:
            folder_upc = upc_like_folder.group('upc')
            existing_upc = self.bad_upc_dict.get(upc_like_folder.group('upc'))

            if existing_upc:
                if not self.duplicate_upcs['bad'].get(folder_upc):
                    self.duplicate_upcs['bad'][folder_upc] = list()

                self.duplicate_upcs['bad'][folder_upc].append(path)
            else:
                self.bad_upc_dict[folder_upc] = \
                    SimpleNamespace(upc_like=upc_like_folder.group('upc'),
                                    extra=upc_like_folder.group('extra'),
                                    tracks=dict())
            return True

        else:
            return False

    @staticmethod
    def validate_audio_file(name):
        """Check if a file name is valid audio."""
        # Any wav file in the correct structure.
        audio_file_pattern = \
            r'(?P<upc>^\d{1,13})\_' \
            r'(?P<vol>\d{1,65535})\_' \
            r'(?P<track>\d{1,65535})\.' \
            r'(?P<ext>(WAV|wav))'

        valid_audio_file = re.match(audio_file_pattern, name)

        return valid_audio_file

    def process_valid_audio_file(self, matches, name, path):
        """Process the audio file."""
        folder_pattern = \
            r'(?P<upc>^\d{1,13})(?P<separator>[\ \-\_]{0,3})(?P<extra>.*)'

        msg = list()

        track_upc = matches.group('upc')
        existing_upc = self.upc_folder_dict.get(track_upc)
        parent = re.match(folder_pattern, path)
        matches_parent = parent and track_upc == parent.group('upc')

        if existing_upc and matches_parent:
            dupe_track = None
            vol = matches.group('vol')
            track = matches.group('track')

            # Check if Volume exists
            vol_dict = self.upc_folder_dict[track_upc][
                'tracks'].get(vol)

            # New Volume
            if not vol_dict:
                self.upc_folder_dict[track_upc]['tracks'][vol] = {
                    track: name.lower()
                }
            else:  # Volume exists
                # Check on dupe track
                dupe_track = self.upc_folder_dict[track_upc]['tracks'][
                    vol].get(track)

            if dupe_track:
                other_path = self.upc_folder_dict[track_upc]['meta'].path
                dupe_upc_exists = self.duplicate_tracks.get(track_upc)

                # Check existing UPC
                if not dupe_upc_exists:
                    self.duplicate_tracks[track_upc] = dict(
                        tracks={
                            vol: {
                                track: [other_path, ]
                            }
                        },
                        name=name.lower(),
                        track_list=[name.lower(), ])
                else:
                    vol_exists = self.duplicate_tracks[track_upc].get(vol)

                    if not vol_exists:
                        self.duplicate_tracks[track_upc]['tracks'][vol] = {
                            track: [other_path, ]
                        }

                self.duplicate_tracks[
                    track_upc]['tracks'][vol][track].append(path)
            else:
                self.upc_folder_dict[track_upc]['tracks'][vol][
                    track] = name.lower()
                # Remove existing upc
                # del(self.upc_folder_dict[track_upc])
        else:
            if not existing_upc:
                msg.append('UPC does not exist at the root level.')
            if not matches_parent:
                msg.append('Track does not match parent folder name/UPC')

            folder_upc = parent.group('upc')

            self.misplaced_audio_files[track_upc] = SimpleNamespace(
                name=name, track_upc=track_upc, folder_upc=folder_upc,
                path=path, msg=msg)

        self.audio_file_list.append(name.lower())

        return True

    @staticmethod
    def validate_broken_file(name):
        # Any possible file audio format separated into wav and non-wav groups
        audio_file_extensions = r'(?P<bad>^.*\.(' \
                                r'aiff|AIFF|mp3|MP3|mp4|MP4|mov|MOV))|' \
                                r'(?P<good>^.*\.(WAV|wav))'

        # Both valid and invalid image formats in groups.
        cover_file_extensions = r'(?P<bad>^.*\.(' \
                                r'jpg|JPG|jpeg|JPEG|gif|GIF|png|PNG))|' \
                                r'(?P<good>^.*\.(TIF|tif|TIFF|tiff))'

        valid_broken_audio_file = re.match(audio_file_extensions, name)

        valid_broken_cover_file = re.match(cover_file_extensions, name)

        return valid_broken_audio_file, valid_broken_cover_file

    def validate_broken_audio_file(self, matches, name, parent):
        """Classify an invalid wav file, if possible."""
        audio_file_upc = r'(?P<upc>^\d{1,13}).*'

        valid_upc = re.match(audio_file_upc, name)
        if not valid_upc:
            upc = 'BAD_UPC'
        else:
            upc = valid_upc.group('upc')

        good = matches.group('good')
        bad = matches.group('bad')

        if good:
            bad_track_dict = self.bad_track_list.get(upc)
            if not bad_track_dict:
                self.bad_track_list[upc] = list()

            self.bad_track_list[upc].append(
                SimpleNamespace(name=good, parent=parent))
        if bad:
            bad_audio_dict = self.bad_audio_extension_list.get(upc)
            if not bad_audio_dict:
                self.bad_audio_extension_list[upc] = list()

            self.bad_audio_extension_list[upc].append(
                SimpleNamespace(name=bad, parent=parent))

    def process_invalid_file(self, name, parent):
        """Process an invalid audio file."""
        self.bad_file_list.append(
            SimpleNamespace(name=name, parent=parent))

    def validate_cover_file(self, name):
        """Check if a filename is valid."""
        # Valid cover file pattern
        cover_file_pattern = r'(?P<upc>^\d{1,13})\_.*\.(tif|TIFF)'

        valid_cover_file = re.match(cover_file_pattern, name)

        return valid_cover_file

    def process_valid_cover_file(self, matches, name, path):
        # Valid cover file pattern
        folder_pattern = \
            r'(?P<upc>^\d{1,13})(?P<separator>[\ \-\_]{1,3})(?P<extra>.*)'

        msg = list()

        cover_upc = matches.group('upc')
        existing_upc = self.upc_folder_dict.get(cover_upc)
        parent = re.match(folder_pattern, path)
        matches_parent = parent and cover_upc == parent.group('upc')

        if existing_upc and matches_parent:
            self.upc_folder_dict[cover_upc]['covers'].append(name.lower())
        else:
            if not existing_upc:
                msg.append('UPC does not exist at the root level.')
            if not matches_parent:
                msg.append('UPC does not match parent folder name')

            folder_upc = parent.group('upc')
            self.misplaced_cover_files[cover_upc] = SimpleNamespace(
                name=name, folder_upc=folder_upc,
                cover_upc=cover_upc, path=path, msg=msg)

        self.cover_file_list.append(name.lower())

        return True

    def validate_broken_cover_file(self, matches, name, parent):
        """Classify an invalid cover file, if possible."""
        cover_file_upc = r'(?P<upc>^\d{1,13}).*'

        valid_upc = re.match(cover_file_upc, name)
        if not valid_upc:
            upc = 'BAD_UPC'
        else:
            upc = valid_upc.group('upc')

        good = matches.group('good')
        bad = matches.group('bad')

        if good:
            bad_cover_dict = self.bad_cover_list.get(upc)
            if not bad_cover_dict:
                self.bad_cover_list[upc] = list()
            self.bad_cover_list[upc].append(SimpleNamespace(
                name=good, parent=parent))
        if bad:
            bad_cover_ext_dict = self.bad_cover_extension_list.get(upc)
            if not bad_cover_ext_dict:
                self.bad_cover_extension_list[upc] = list()

            self.bad_cover_extension_list[upc].append(SimpleNamespace(
                name=bad, parent=parent))

    def get_path_for_wav(self, name, separator=None):
        if not separator:
            separator = '_'

        split_name = name.split(separator)
        upc = split_name[0]
        vol = split_name[1]
        track = split_name[2].split('.')[0]

        try:
            file = self.upc_folder_dict[upc]['tracks'][vol][track]
        except Exception as e:
            file = None
            logger.bind(file=True).error(str(e))
        if file:
            return self.upc_folder_dict[upc]['meta'].path
        else:
            return None

    def get_path_for_cover(self, name):

        split_name = name.split('.')
        upc = split_name[0]

        if name in self.upc_folder_dict[upc]['covers']:
            return self.upc_folder_dict[upc]['meta'].path
        else:
            return None


class MetadataAnalysis(IngestionAnalysis):
    def validate_upc(self, path, pattern=None):
        """Check if folder matches valid upc structure

        Args:
            path:

        Returns:

        """
        # Pattern for valid upc-folders
        if not pattern:
            pattern = \
                r'(?P<upc>^\d{1,13})'

        upc_folder = re.match(pattern, path)

        if upc_folder:
            folder_upc = upc_folder.group('upc')
            existing_upc = self.upc_folder_dict.get(folder_upc)

            if existing_upc:
                if not self.duplicate_upcs['good'].get(folder_upc):
                    self.duplicate_upcs['good'][folder_upc] = 1

                self.duplicate_upcs['good'][folder_upc] += 1

            else:
                self.upc_folder_dict[folder_upc] = dict(
                    meta=SimpleNamespace(
                        upc=folder_upc,
                        path=path),
                    tracks=dict(),
                    covers=list())
            return True
        else:
            return False

    # TODO Add the ability to confirm the track name vs the vol, track columns
    def process_valid_audio_file(self, matches, name):
        """Process the audio file."""

        msg = list()
        dupe_track = None
        track_upc = matches.group('upc')
        existing_upc = self.upc_folder_dict.get(track_upc)

        if existing_upc:
            vol = matches.group('vol')
            track = matches.group('track')

            # Check if Volume exists
            vol_dict = self.upc_folder_dict[track_upc][
                'tracks'].get(vol)

            # New volume
            if not vol_dict:
                self.upc_folder_dict[track_upc]['tracks'][vol] = {
                    track: name.lower()
                }
            else:  # Volume exists
                # Check on dupe track
                dupe_track = self.upc_folder_dict[track_upc]['tracks'][
                    vol].get(track)

            if dupe_track:
                dupe_upc_exists = self.duplicate_tracks.get(track_upc)

                # Check existing UPC
                if not dupe_upc_exists:
                    self.duplicate_tracks[track_upc] = dict(
                        tracks={
                            vol: {
                                track: 1
                            }
                        },
                        name=name.lower(),
                        track_list=[name.lower(), ])
                else:
                    vol_exists = self.duplicate_tracks[track_upc].get(vol)

                    if not vol_exists:
                        self.duplicate_tracks[track_upc]['tracks'][vol] = {
                            track: 1
                        }

                self.duplicate_tracks[
                    track_upc]['tracks'][vol][track] += 1

            else:
                self.upc_folder_dict[track_upc]['tracks'][vol][
                    track] = name.lower()
        else:
            msg.append('UPC does not exist in the upc column.')

            self.misplaced_audio_files[track_upc] = SimpleNamespace(
                name=name.lower(), track_upc=track_upc, msg=msg)

        self.audio_file_list.append(name.lower())

        return True

    @staticmethod
    def validate_broken_file(name):
        # Any possible file audio format separated into wav and non-wav groups
        audio_file_extensions = r'(?P<bad>^.*\.(' \
                                r'aiff|AIFF|mp3|MP3|mp4|MP4|mov|MOV))|' \
                                r'(?P<good>^.*\.(WAV|wav))'

        valid_broken_audio_file = re.match(audio_file_extensions, name)

        return valid_broken_audio_file

    def validate_broken_audio_file(self, matches, name):
        """Classify an invalid wav file, if possible."""
        audio_file_upc = r'(?P<upc>^\d{1,13}).*'

        upc = name.split('.')[0]
        good = matches.group('good')
        bad = matches.group('bad')

        if good:
            bad_track_dict = self.bad_track_list.get(upc)
            if not bad_track_dict:
                self.bad_track_list[upc] = list()

            self.bad_track_list[upc].append(
                SimpleNamespace(name=good))
        if bad:
            valid_upc = re.match(audio_file_upc, name)
            if not valid_upc:
                upc = 'BAD_UPC'

            bad_audio_dict = self.bad_audio_extension_list.get(upc)
            if not bad_audio_dict:
                self.bad_audio_extension_list[upc] = list()

            self.bad_audio_extension_list[upc].append(
                SimpleNamespace(name=bad))

    def process_invalid_file(self, name):
        """Process an invalid audio file."""
        self.bad_file_list.append(SimpleNamespace(name=name))


def get_metadata_structure(metadata_upc_list, metadata_track_list):
    """Checks a list to see if the strings contained therein all conform to
    an optionally passed format.

    Args:
        elements: (list) A list of elements to check
        pattern: (str) Optional. A regex pattern to use for comparisons.

    Returns:
        IngestionAnalysis: An IngestionAnalysis object.
    """

    ma = MetadataAnalysis()

    for upc in metadata_upc_list:
        if not ma.validate_upc(upc):
            logger.bind(file=True).info('UPC {} is not valid.', upc)
            logger.bind(csv_only=True).info('{}|NA|Metadata|Invalid UPC', upc)

            if not ma.validate_broken_folder(upc):
                logger.bind(file=True).info(
                    'UPC does not begin with a number.')
                logger.bind(csv_only=True).info(
                    '{}|NA|Metadata|UPC does not begin with a number', upc)
            else:
                ma.bad_folder_list.append(upc)

    for name in metadata_track_list:
        valid_audio_file = ma.validate_audio_file(name)
        valid_cover_file = ma.validate_cover_file(name)

        if valid_audio_file:
            ma.process_valid_audio_file(valid_audio_file, name)
        elif valid_cover_file:
            ma.process_valid_cover_file(valid_cover_file, name)
        else:
            broken_audio_file = ma.validate_broken_file(
                name)

            if broken_audio_file:
                ma.validate_broken_audio_file(
                    broken_audio_file, name)
            else:
                ma.process_invalid_file(name)

    return ma


def get_asset_folder_structure(root):
    """Get a report about the root path.

    Args:
        root: (str) The root path to traverse down from.

    Returns:
        dict: A multi-level dict representing the path and file structure..
    """
    # Create Folder Analysis object
    fa = IngestionAnalysis()

    for path, subdirs, files in os.walk(root):
        current_dir = path.split('/')[-1]
        # Check subdirs in folder
        for sub in subdirs:
            if not fa.validate_upc_folder(sub):
                logger.bind(file=True).info('Folder {} is not valid.', sub)
                logger.bind(csv_only=True).info(
                    '{}|NA|Data source|Invalid UPC folder', sub, )
                if not fa.validate_broken_folder(sub):
                    logger.bind(file=True).info(
                        'Folder does not begin with a number.')
                    logger.bind(csv_only=True).info(
                        '{}|NA|{}|Folder does not begin with a number',
                        sub, path)

                elif root == path:
                    fa.bad_folder_list.append(sub)

        for name in files:
            # Check if file is a garbage file
            ignore = False
            for pat in IGNORE_FILES:
                if fnmatch(name, pat):
                    ignore = True

            if ignore:
                continue

            # process non-garbage files.
            valid_audio_file = fa.validate_audio_file(name)
            valid_cover_file = fa.validate_cover_file(name)

            if valid_audio_file:
                fa.process_valid_audio_file(
                    valid_audio_file, name, current_dir)
            elif valid_cover_file:
                fa.process_valid_cover_file(
                    valid_cover_file, name, current_dir)
            else:
                broken_audio_file, broken_cover_file = fa.validate_broken_file(
                    name)

                if broken_audio_file:
                    fa.validate_broken_audio_file(
                        broken_audio_file, name, current_dir)
                elif broken_cover_file:
                    fa.validate_broken_cover_file(
                        broken_cover_file, name, current_dir)
                else:
                    fa.process_invalid_file(name, current_dir)

    return fa


def get_tracks_and_upcs_from_data_source(data_source_folder, separator=None,
                                         position=None):
    """Get the names of wav files and folders in given folder path.

    Args:
        data_source_folder: (str) the path to the data_source
        separator: (str) the string to be used to extract the UPC from the
        folder name. Defaults to ' ' (a single space)
        position: (int) The index of the UPC in the string as split() by
        `separator`. Defaults to 0.

    Returns:
        tuple: (list, list) A list of wav files in the subdirectories of the
        given root path, and a list of folder-based UPCS.
    """
    if not position:
        position = 0

    if not separator:
        separator = ' '

    # Report to user
    logger.info(
        'Getting list of UPC\'s from the top level of {}.', data_source_folder)

    # Get list of files in data source
    data_source_upc_list = get_all_dirs_in_subdirs(data_source_folder,
                                                   root_only=True)

    # Slice off UPCS
    data_source_upc_list = [x.split(separator)[position]
                            for x in data_source_upc_list]

    valid_upc = [x for x in data_source_upc_list if represents_int(x)]

    # Report to user
    logger.info('{} UPC candidates found in Data Source.',
                len(data_source_upc_list))
    logger.info('{} UPC candidates appear valid.',
                len(valid_upc))
    logger.info('Getting list of WAV files in all subdirs of {}.',
                data_source_folder)
    logger.info('Be patient. This may take a while....')

    # TODO - Update to check if the UPC part of the files in each directory
    #  match the parent folder UPC.
    data_source_track_list = get_all_files_in_subdirs(
        data_source_folder, pattern='*.wav')

    # Report to user
    logger.info('{} WAV files found in Data Source.',
                len(data_source_track_list))

    # Return lists
    return data_source_track_list, data_source_upc_list


def report_duplicates(elements):
    """Log to warning all found duplicates in a list.

    Args:
        elements: (list) A list of elements.

    """
    dupe_list = get_duplicates_in_list(elements)

    for x, cnt in dupe_list.items():
        logger.info('{}: {} occurrences'.format(x, cnt))


def report_comparisons(list_1, list_2, list_name_1=None, list_name_2=None):
    """Reports on the comparison between two lists.

    Args:
        list_1: (list) A list of elements
        list_2: (list) A second list of elements
        list_name_1: Optional. A name associated with the first list.
        Defaults to 'first list'
        list_name_2: Optional. A name associated with the second list.
    """
    if not list_name_1:
        list_name_1 = 'first list'

    if not list_name_2:
        list_name_2 = 'second list'

    comparisons = compare_lists(list_1, list_2)

    logger.info('{} elements shared between {} and {}',
                len(comparisons['both']), list_name_1, list_name_2)

    logger.info('{} elements exist only in {}',
                len(comparisons['list_1_only']), list_name_1)

    for el in comparisons['list_1_only']:
        logger.info('{}', el)

    logger.info('{} elements exist only in {}',
                len(comparisons['list_2_only']), list_name_2)

    for el in comparisons['list_2_only']:
        logger.info('{}', el)

    return comparisons


def analyze_sources(data_source, metadata_source):
    """Report on analysis."""
    image_extension = 'tif'
    # Metadata checks
    topic = 'Metadata audio tracks missing in data source'
    logger.bind(file=True).info(separator_long, topic)
    msg = list()
    for file in metadata_source.audio_file_list:
        file_validation = IngestionAnalysis.validate_audio_file(file)
        if not file_validation:
            logger.bind(file=True).info('{} is not a valid file', file)
            logger.bind(csv_only=True).info(
                'Topic|NA|{}|Metadata|Invalid file', file)
            continue
        else:
            upc = file_validation.group('upc')

        if file not in data_source.audio_file_list:
            msg.append('{} is missing in the data source.'.format(file))

            logger.bind(csv_only=True).info(
                '{}|{}|{}|Metadata|Missing in data source', topic, upc, file)

            orig_file = file
            while file[0] == '0':
                file = file[1:]
                if file in data_source.audio_file_list:
                    msg.append(
                        '{} has leading zeroes. {} found in the data '
                        'source.'.format(orig_file, file))

                    path = data_source.get_path_for_wav(file)
                    logger.bind(csv_only=True).info(
                        '{}|{}|{}|"{}"|File has leading zeroes. '
                        'Similar file {} found in data source.',
                        topic, upc, orig_file, path, file)

                    msg.append('Found in path {}'.format(path))

            file = orig_file

            while len(file.split('_')[0]) < 13:
                file = '0' + file
                if file in data_source.audio_file_list:
                    msg.append(
                        '{} is missing leading zeroes. '
                        '{} found in the data'.format(orig_file, file))
                    path = data_source.get_path_for_wav(file)
                    logger.bind(csv_only=True).info(
                        '{}|{}|{}|"{}"|File is missing leading zeroes. '
                        'Similar file {} found in data source.',
                        topic, upc, orig_file, path, file)

                    msg.append('Found in path {}'.format(path))

    if len(msg):
        logger.bind(file=True).info(
            '{} files missing in the data source.', len(msg))

        for message in msg:
            logger.bind(file=True).info(message)

    # Data source checks
    topic = 'Data Source audio tracks missing in metadata'
    logger.bind(file=True).info('')
    logger.bind(file=True).info(separator_long, topic)
    msg = list()
    for file in data_source.audio_file_list:
        file_validation = IngestionAnalysis.validate_audio_file(file)
        if not file_validation:
            logger.bind(file=True).info('{} is not a valid file', file)
            logger.bind(csv_only=True).info(
                '{}|NA|{}|Metadata|Invalid File', topic, file)
            continue
        else:
            upc = file_validation.group('upc')

        if file not in metadata_source.audio_file_list:
            path = data_source.get_path_for_wav(file)
            msg.append('{} is missing in the metadata.'.format(file))
            logger.bind(csv_only=True).info(
                '{}|{}|{}|"{}"|Missing in metadata',
                topic, upc, file, path or 'NA')

            orig_file = file
            while file[0] == '0':
                file = file[1:]
                if file in metadata_source.audio_file_list:
                    msg.append('{} has leading zeroes. {} found in the data '
                               'source.'.format(orig_file, file))

                    logger.bind(csv_only=True).info(
                        '{}|{}|{}|"{}"|File has leading zeroes. '
                        'Similar file {} found in data source.',
                        topic, upc, orig_file, path or 'NA', file)

                    path = data_source.get_path_for_wav(file)

                    if path:
                        msg.append('Found in path {}'.format(path))
                    else:
                        msg.append('Path for file {} not found'.format(file))

            file = orig_file

            while len(file.split('_')[0]) < 13:
                file = '0' + file
                if file in metadata_source.audio_file_list:
                    msg.append('{} is missing leading zeroes. '
                               '{} found in the data'.format(orig_file, file))

                    logger.bind(csv_only=True).info(
                        '{}|{}|{}|"{}"|File is missing leading zeroes. '
                        'Similar file {} found in data source.',
                        topic, upc, orig_file, path, file)

                    path = data_source.get_path_for_wav(file)

                    if path:
                        msg.append('Found in path {}'.format(path))
                    else:
                        msg.append('Path for file {} not found'.format(file))
                    msg.append('Found in path {}'.format(path))

    if len(msg):
        logger.bind(file=True).info(
            '{} files missing in the data source.', len(msg))

        for message in msg:
            logger.bind(file=True).info(message)

    # Metadata checks
    topic = 'Metadata covers missing in data source'
    logger.bind(file=True).info('')
    logger.bind(file=True).info(separator_long, topic)
    for upc, info in metadata_source.upc_folder_dict.items():
        if not data_source.upc_folder_dict.get(upc):
            logger.bind(file=True).info(
                'UPC {} is missing in the data source.')
            logger.bind(csv_only=True).info(
                '{}|{}|NA|Metadata|UPC is missing in data source',
                topic, upc)

        elif not len(data_source.upc_folder_dict[upc]['covers']):
            logger.bind(file=True).info(
                'File {}.{} is missing in the data source.',
                info['meta'].path, image_extension)
            logger.bind(csv_only=True).info(
                '{}|{}|{}.{}|Metadata|File is missing',
                topic, upc, upc, image_extension)

    # Data source checks
    topic = 'Data Source covers missing in metadata'
    logger.bind(file=True).info('')
    logger.bind(file=True).info(separator_long, topic)
    if not len(data_source.cover_file_list):
        logger.bind(file=True).info(
            'All covers are missing or invalid in the data source.')
    else:
        for file in data_source.cover_file_list:
            if file.split('.')[0] \
                    not in metadata_source.upc_folder_dict.keys():
                path = data_source.get_path_for_wav(file)
                logger.bind(file=True).info(
                    '{} is missing in the metadata.', file)
                logger.bind(csv_only=True).info(
                    '{}|{}|NA|"{}"|File is missing', topic, file, path)

    # Report on misplaced audio
    topic = 'Misplaced Audio'
    logger.bind(file=True).info('')
    logger.bind(file=True).info(separator_long, topic)
    logger.bind(file=True).info(
        '{} Misplaced audio files found:',
        len(data_source.misplaced_audio_files))

    for track_upc, error_info in data_source.misplaced_audio_files.items():
        logger.bind(file=True).info(
            '{} --------', track_upc)
        logger.bind(file=True).info(
            '{} found in folder {}', error_info.name, error_info.path)
        for msg in error_info.msg:
            logger.bind(file=True).info(msg)
            logger.bind(csv_only=True).info(
                '{}|{}|{}|"{}"|{}', topic, track_upc, error_info.name,
                error_info.path, msg)
        logger.bind(file=True).info('')

    # Report on duplicate UPC folders
    topic = 'Duplicate UPC\'s in data source'
    logger.bind(file=True).info('')
    logger.bind(file=True).info(separator_long, topic)
    logger.bind(file=True).info(
        '{} valid duplicate UPC\'s found on data source.',
        len(data_source.duplicate_upcs['good']))

    for upc, folders in data_source.duplicate_upcs['good'].items():
        logger.bind(file=True).info(separator_short, upc)

        for el in folders:
            logger.bind(file=True).info('{}', el)
            logger.bind(csv_only=True).info(
                '{}|{}|NA|"{}"|Duplicate valid UPC', topic, upc, el)
        logger.bind(file=True).info('')

    # Report on duplicate UPC folders
    topic = 'Duplicates UPC\'s on metadata'
    logger.bind(file=True).info('')
    logger.bind(file=True).info(separator_long, topic)
    logger.bind(file=True).info(
        '{} valid duplicate UPC\'s found in metadata.',
        len(metadata_source.duplicate_upcs['good']))

    for upc, count in metadata_source.duplicate_upcs['good'].items():
        logger.bind(file=True).info(separator_short, upc)

        logger.bind(file=True).info('{} occurrences of duplicate UPC', count)
        logger.bind(csv_only=True).info(
            '{}|{}|NA|Metadata|{} occurrences of duplicate valid UPC',
            topic, upc, count)

    #  logger.bind(csv=True).info('Topic|UPC|File|Location|Note')

    # Report on duplicate tracks
    topic = 'Duplicate Tracks in Data Source'
    logger.bind(file=True).info('')
    logger.bind(file=True).info(separator_long, topic)
    logger.bind(file=True).info(
        '{} UPC\'s with duplicate tracks found in data source.',
        len(data_source.duplicate_tracks))

    for upc, info in data_source.duplicate_tracks.items():
        logger.bind(file=True).info(separator_short, upc)

        for vol, tracks in info['tracks'].items():
            for track, track_list in tracks.items():
                for item in track_list:
                    logger.bind(file=True).info(
                        'Volume {}, Track {} - {}', vol, track, item)
                    logger.bind(csv_only=True).info(
                        '{}|{}|{}|"{}"|Duplicate track', topic, upc,
                        info['name'], item)

        logger.bind(file=True).info('')

    # Report on bad audio extension
    topic = 'Bad Audio Extensions in Data Source'
    logger.bind(file=True).info('')
    logger.bind(file=True).info(separator_long, topic)
    logger.bind(file=True).info('{} audio files found with bad audio '
                                'extensions in data source.',
                                len(data_source.bad_audio_extension_list))

    for upc, error_list in data_source.bad_audio_extension_list.items():
        logger.bind(file=True).info(separator_short, upc)

        for error_info in error_list:
            logger.bind(file=True).info(
                '{}/{}', error_info.parent, error_info.name)
            logger.bind(csv_only=True).info(
                '{}|{}|{}|"{}"|Bad audio format',
                topic, upc, error_info.name, error_info.parent)

        logger.bind(file=True).info('')

    # Report on bad cover extension
    topic = 'Bad Cover Extensions in Data Source'
    logger.bind(file=True).info('')
    logger.bind(file=True).info(separator_long, topic)
    logger.bind(file=True).info('{} cover files found with bad image '
                                'extensions in data source.',
                                len(data_source.bad_cover_extension_list))
    for upc, error_info in data_source.bad_cover_extension_list.items():
        logger.info(separator_short, upc)

        for err in error_info:
            logger.bind(file=True).info(
                '{}/{}', err.parent, err.name)
            logger.bind(csv_only=True).info(
                '{}|{}|{}|"{}"|Bad cover extension',
                topic, upc, err.name, err.parent)

        logger.bind(file=True).info('')

    # Report on bad audio files
    logger.info('')
    topic = 'Bad Audio File List in Data Source'
    logger.bind(file=True).info(separator_long, topic)
    logger.info('{} UPC\'s with bad audio files found in data source.',
                len(data_source.bad_track_list))
    for upc, info in data_source.bad_track_list.items():
        for file in info:
            logger.bind(file=True).info('{}/{}', file.parent, file.name)
            logger.bind(csv_only=True).info(
                '{}|{}|{}|"{}"|Bad audio file',
                topic, upc, file.name, file.parent)

    # Report on bad cover files
    logger.bind(file=True).info('')
    topic = 'Bad Cover File List'
    logger.bind(file=True).info(separator_long, topic)
    logger.bind(file=True).info('{} UPC\'s with bad cover files found in data '
                                'source.', len(data_source.bad_cover_list))

    for upc, info in data_source.bad_cover_list.items():
        logger.bind(file=True).info('{}/{}', info.parent, info.name)
        for file in info:
            logger.bind(file=True).info('{}/{}', file.parent, file.name)
            logger.bind(csv_only=True).info(
                '{}|{}|{}|"{}"|Bad cover file',
                topic, upc, file.name, file.parent)

    # Report on bad files
    topic = 'Bad Files in Data Source'
    logger.bind(file=True).info('')
    logger.bind(file=True).info(separator_long, topic)
    logger.info('{} bad files found in data source.',
                len(data_source.bad_file_list))
    for file in data_source.bad_file_list:
        logger.info('{}/{}', file.parent, file.name)
        logger.bind(csv_only=True).info(
            '{}|NA|{}|"{}"|Bad file', topic, file.name, file.parent)

    # Report on UPC's
    topic = 'Invalid tracks in data source'
    logger.bind(file=True).info('')
    logger.bind(file=True).info(separator_long, topic)
    logger.bind(file=True).info(
        '{} valid UPC\'s found.',
        len(data_source.upc_folder_dict))
    logger.bind(file=True).info(
        'Checking tracks from data source are present in metadata.')

    valid_track_msgs = dict()

    for upc, info in data_source.upc_folder_dict.items():
        msg = list()
        failed = False

        if not len(info['tracks']):
            error = 'No valid tracks found in data source.'
            msg.append(error)
            logger.bind(csv_only=True).info('{}|{}|UPC|"{}"|{}', topic, upc,
                                            info['meta'].path, error)
            failed = True
        else:
            for vol, tracks in info['tracks'].items():
                track_order = sorted(tracks.keys())
                if not check_consecutive(track_order):
                    error = 'Tracks in folder are not consecutive.'
                    msg.append(error)
                    logger.bind(csv_only=True).info(
                        '{}|{}|UPC|"{}"|{}',
                        topic, upc, info['meta'].path, error)
                    failed = True

        if failed:
            logger.bind(file=True).info(separator_short, upc)

            for message in msg:
                logger.bind(file=True).info(message)

            logger.bind(file=True).info('')
            continue

        if not metadata_source.upc_folder_dict.get(upc):
            error = 'UPC is not present in metadata.'
            msg.append(error)
            logger.bind(csv_only=True).info(
                '{}|{}|UPC|"{}"|{}', topic, upc, info['meta'].path, error)
            continue

        for vol, tracks in info['tracks'].items():
            if not metadata_source.upc_folder_dict[upc]['tracks'].get(vol):
                error = 'Volume {} is not present in metadata.'.format(vol)
                msg.append(error)
                logger.bind(csv_only=True).info(
                    '{}|{}|UPC|"{}"|{}', topic, upc, info['meta'].path, error)
            else:
                for track, name in data_source.upc_folder_dict[
                        upc]['tracks'][vol].items():
                    if track not in metadata_source.upc_folder_dict[
                            upc]['tracks'][vol].keys():
                        error = 'Volume {}, Track {} is not present in ' \
                                'metadata'.format(vol, track)
                        msg.append(error)
                        logger.bind(csv_only=True).info(
                            '{}|{}|{}|"{}"|{}',
                            topic, upc, name, info['meta'].path, error)
                        failed = True

                if not failed:
                    valid_track_msgs[upc] = list()
                    logger.bind(csv_only=True).info(
                        'Valid Audio Releases|{}|UPC|"{}"|Release audio '
                        'structure is valid', upc, info['meta'].path)
                    for track, name in data_source.upc_folder_dict[
                            upc]['tracks'][vol].items():
                        valid_track_msgs[upc].append(name)
                        logger.bind(csv_only=True).info(
                            'Valid Audio Tracks|{}|{}|"{}"|Track is valid',
                            upc, name, info['meta'].path)

        if len(msg):
            logger.bind(file=True).info(separator_short, upc)

        for message in msg:
            logger.bind(file=True).info('{}', message)
        if len(msg):
            logger.bind(file=True).info('')

        # Report on UPC's
    topic = 'Invalid tracks in Metadata'
    logger.bind(file=True).info('')
    logger.bind(file=True).info(separator_long, topic)
    logger.bind(file=True).info('{} valid UPC\'s found.',
                                len(data_source.upc_folder_dict))
    logger.bind(file=True).info(
        'Checking tracks from in metadata are present in data source.')

    for upc, info in metadata_source.upc_folder_dict.items():
        msg = list()
        # logger.bind(file=True).info('{}', upc)
        if not len(info['tracks']):
            error = 'No valid tracks found in metadata.'
            msg.append(error)
            logger.bind(csv_only=True).info(
                '{}|{}|UPC|Metadata|{}', topic, upc, error)
        else:
            for vol, tracks in info['tracks'].items():
                track_order = sorted(tracks.keys())
                if not check_consecutive(track_order):
                    error = 'Tracks in folder are not consecutive.'
                    msg.append(error)
                    logger.bind(csv_only=True).info(
                        '{}|{}|UPC|Metadata|{}', topic, upc, error)

        for vol, tracks in info['tracks'].items():
            if not data_source.upc_folder_dict.get(upc):
                error = 'UPC is not present in data source.'
                msg.append(error)
                logger.bind(csv_only=True).info(
                    '{}|{}|UPC|Metadata|{}', topic, upc, error)

            elif not data_source.upc_folder_dict[upc]['tracks'].get(vol):
                error = 'Volume {} is not present in data source.'.format(vol)
                msg.append(error)
                logger.bind(csv_only=True).info(
                    '{}|{}|UPC|Metadata|{}', topic, upc, error)
            else:
                for track, name in metadata_source.upc_folder_dict[
                        upc]['tracks'][vol].items():
                    if track not in data_source.upc_folder_dict[
                            upc]['tracks'][vol].keys():
                        error = 'Volume {}, Track {} is not present in ' \
                                'data source'.format(vol, track)
                        msg.append(error)
                        logger.bind(csv_only=True).info(
                            '{}|{}|{}|Metadata|{}', topic, upc, name, error)

        if len(msg):
            logger.bind(file=True).info(separator_short, upc)

        for message in msg:
            logger.bind(file=True).info('{}', message)

        if len(msg):
            logger.bind(file=True).info('')

    topic = 'Valid Assets on data source'
    logger.bind(file=True).info('')
    logger.bind(file=True).info(separator_long, topic)

    for upc, tracks in valid_track_msgs.items():
        # for track in tracks:
        logger.bind(file=True).info('{} is valid', upc)
    logger.bind(file=True).info('')

    # TODO - OUTPUT bash script-like thing with accurate files.


def main():
    # TODO - Use XLSX file instead of text files
    # Define vars
    track_input_file_name = '/Users/lvona/Documents/python_projects/ripper' \
                            '-feeder/data/INT-484_full_track_list_from_' \
                            'metadata_spreadsheet.txt'

    upc_input_file_name = '/Users/lvona/Documents/python_projects/ripper' \
                          '-feeder/data/INT-484_full_upc_list_from_metadata_' \
                          'spreadsheet.txt'

    data_source_folder = '/Volumes/SuburbanCat/The Orchard'

    # data_source_folder = '/Volumes/ripped_assets/ripper_feeder_staging' \
    #                      '/Suburban/unconformed'

    # output_file = '/Users/lvona/Documents/python_projects/ripper-feeder' \
    #               '/validation_report_suburban_2020.txt'

    # Get list of tracks from the metadata infile.
    with open(track_input_file_name) as f:
        metadata_track_list = f.readlines()

    # Clean off trailing '\n's
    metadata_track_list = [x.rstrip('\n').lower() for x in metadata_track_list]

    # TODO - Check all track files fit 'UPC_VOL_TRACK.WAV' pattern
    # if get_metadata_structure(data_source_folder):
    #     logger.info('Metadata has tracks with bad structure.')
    #     report_bad_structure(metadata_track_list)
    #     metadata_track_list = remove_bad_tracks(metadata_track_list)

    # Get list of UPC's from the metadata infile.
    with open(upc_input_file_name) as f:
        metadata_upc_list = f.readlines()

    metadata_upc_list = [x.rstrip('\n') for x in metadata_upc_list]

    # De-dupe UPC's
    metadata_upc_list = dedupe_list(metadata_upc_list)

    logger.bind(csv_only=True).info('Topic|UPC|Track|Location|Notes')

    # TODO - Update to access S3 if needed.
    # data_source_track_list, data_source_upc_list = \
    #     get_tracks_and_upcs_from_data_source(data_source_folder)

    logger.bind(file=True).info('Analyzing metadata....')
    all_metadata = get_metadata_structure(metadata_upc_list,
                                          metadata_track_list)

    logger.bind(file=True).info('Analyzing data source....')
    logger.bind(file=True).info('Be patient. This may take a while....')
    all_data = get_asset_folder_structure(data_source_folder)

    # Report UPC length compare
    metadata_upc_list_len = len(metadata_upc_list)
    logger.bind(file=True).warning('{} unique UPC\'s appear in Metadata',
                                   metadata_upc_list_len)
    logger.bind(csv_only=True).info(
        'Summary|NA|NA|Metadata|{} unique UPCs found',
        metadata_upc_list_len)

    datasource_upc_list_len = len(all_data.upc_folder_dict.keys())

    logger.bind(file=True).warning('{} unique UPC\'s appear in Data Source',
                                   datasource_upc_list_len)
    logger.bind(csv_only=True).info(
        'Summary|NA|NA|Data Source|{} appear UPCs found',
        datasource_upc_list_len)

    # Report UPC comparisons
    upc_comparisons = report_comparisons(
        metadata_upc_list, list(all_data.upc_folder_dict.keys()),
        'Metadata UPC List', 'Data Source UPC List')

    # Report track comparisons
    track_comparisons = report_comparisons(
        metadata_track_list, all_data.audio_file_list,
        'Metadata Track List', 'Data Source Track List')

    for source, items in upc_comparisons.items():
        if source == 'list_1_only':
            logger.bind(csv_only=True).info(
                'Summary|NA|NA|Metadata|{} unique UPCs appear in metadata '
                'only', len(items))
            for item in items:
                logger.bind(csv_only=True).info(
                    'Summary|{}|UPC|Metadata|UPC is not present in data '
                    'source', item)

        if source == 'list_2_only':
            logger.bind(csv_only=True).info(
                'Summary|NA|NA|Data Source|{} unique UPCs appear in data '
                'source only', len(items))
            for item in items:
                logger.bind(csv_only=True).info(
                    'Summary|{}|UPC|Data Source|UPC is not present in '
                    'metadata.', item)

        if source == 'both':
            logger.bind(csv_only=True).info(
                'Summary|NA|UPC|BOTH|{} unique UPCs appear in both sources',
                len(items))
            for item in items:
                logger.bind(csv_only=True).info(
                    'Summary|{}|UPC|BOTH|UPC is found in both sources',
                    item)

    for source, items in track_comparisons.items():
        if source == 'list_1_only':
            logger.bind(csv_only=True).info(
                'Summary|NA|NA|Metadata|{} tracks appear in metadata '
                'only', len(items))
            for item in items:
                logger.bind(csv_only=True).info(
                    'Summary|{}|{}|Metadata|Track is not present in data '
                    'source', item.split('_')[0], item)

        if source == 'list_2_only':
            logger.bind(csv_only=True).info(
                'Summary|NA|NA|Data Source|{} unique tracks appear in data '
                'source only', len(items))
            for item in items:
                logger.bind(csv_only=True).info(
                    'Summary|{}|{}|Data Source|Track is not present in '
                    'metadata', item.split('_')[0], item)

        if source == 'both':
            logger.bind(csv_only=True).info(
                'Summary|NA|NA|BOTH|{} tracks appear in both sources',
                len(items))
            for item in items:
                logger.bind(csv_only=True).info(
                    'Summary|{}|{}|BOTH|Track is found in both sources',
                    item.split('_')[0], item)

    logger.bind(file=True).info('Comparing sources.')
    analyze_sources(all_data, all_metadata)


if __name__ == '__main__':
    main()