"""Reusable Garcon Tasks related to a feed's status."""
import os
import re

import boto3
from boto3.dynamodb.conditions import Attr, Key
from garcon import task
from garcon_contrib.dynamo_feed_status import config as feed_status_config
from garcon_contrib.dynamo_feed_status import garcon_feed_status

from feed_ingestion.flows import helpers
from feed_ingestion.tasks.overall_status_tasks import \
    set_overall_status_enhanced
from feed_ingestion.util import task_status
from feed_ingestion.util.aws import s3 as s3utils


@task.decorate(timeout=1000)
def update_feed_file_status(activity, feed_name, date, files,
                            s3_path, report_type=None,
                            contexts_config=dict(), licensor=None):
    """Update Feed Status whether all non-optional files were downloaded.

    (Currently used only in apple_music_streams workflow).

    Args:
        activity (ActivityWorker): The activity worker.
        feed_name (str): Name of the feed in the status table.
        date (str): Reporting Date (YYYY-MM-DD).
        files (list): List of filenames that should be downloaded for the
            feed_name & date params.
        s3_path (str): S3 path,
            e.g. s3://cucumbers/AppleMusicStreams/archives/2020-05-11/.
        report_type (str): Report name, e.g. amContainer.
        contexts_config (dict): config for required and optional contexts.
        licensor (str): Name of the licensor.

    Returns:
        dict: Adds a 'file_status' entry to the context indicating whether
            all required files for the supplied feed & date were Downloaded or
            Not Available.
    """
    item = task_status.get_item(feed_name, date)
    missing_files = []
    for file in files:
        activity.logger.info('Checking feed status for {}'.format(file))
        file_atr = garcon_feed_status.get_attribute_name_for_file_status(
            file, date)
        if item.get(file_atr, '') != garcon_feed_status.STATUS_DOWNLOADED:
            missing_files.append(s3_path + file)
            # set STATUS_NOT_AVAILABLE only if missing file is required
            if not file_is_optional(
                    report_type, contexts_config, file, licensor):
                activity.logger.info('Required file is missing: {}'
                                     .format(file))
                set_overall_status_enhanced(
                    feed_name, date, garcon_feed_status.STATUS_NOT_AVAILABLE,
                    activity)
    if missing_files:
        # set missing file no meter if they are optional or not
        # will use these field in check_available_reports task
        garcon_feed_status.set_missing_files(feed_name, date, missing_files)
        for mf in missing_files:
            # return STATUS_NOT_AVAILABLE only if missing file is required
            if not file_is_optional(
                    report_type, contexts_config, mf, licensor):
                return {'file_status': garcon_feed_status.STATUS_NOT_AVAILABLE}

    new_contexts = task_status.get_newcontexts(feed_name, date)

    # only update status if we are still waiting to download the files
    if (not garcon_feed_status.get_overall_status(feed_name, date) or
            garcon_feed_status.get_overall_status(feed_name, date) ==
            garcon_feed_status.STATUS_NOT_AVAILABLE or
            new_contexts):
        set_overall_status_enhanced(
            feed_name, date, garcon_feed_status.STATUS_DOWNLOADED, activity)
        task_status.mark_completed_task(feed_name, date, 'reporter_to_s3')
        activity.logger.info(f'Set Downloaded status for {feed_name}')

    return {'file_status': garcon_feed_status.STATUS_DOWNLOADED}


@task.decorate(timeout=1000)
def update_feed_s3_file_status(
        activity, s3_path, file_names, feed_name=None, date=None):
    """Check workflow validity.

    Check workflow validity by checking if dependent files are in S3 and
    updates the feed status accordingly.

    Args:
         activity (ActivityWorker): The activity worker.
         s3_path (str): S3 'folder' or S3 key without the file name.
         file_names (list): List of files to look for in S3.
         feed_name (str): Name of the feed in the status table.
         date (str): Date info for report.

    Returns:
        dict: 'file_status' entry to the context indicating whether
            all files for the feed & date are present or unavailable.
    """
    missing_files = []
    for file in file_names:
        activity.logger.info(
            'Checking file status for {} {}'.format(s3_path, file))
        # Check if key is in s3 bucket
        file_name = ''.join([s3_path, file])
        if not helpers.check_s3_key_exist(file_name):
            activity.logger.warning(f'MISSING {s3_path}{file}')
            missing_files.append(file_name)
        else:
            garcon_feed_status.set_status(
                feed_name, date, file_name,
                status=garcon_feed_status.STATUS_DOWNLOADED)
    if missing_files:
        if feed_name is not None and date is not None:
            set_overall_status_enhanced(
                feed_name, date, garcon_feed_status.STATUS_NOT_AVAILABLE,
                activity)
            garcon_feed_status.set_missing_files(
                feed_name, date, missing_files)
        return {
            'file_status': garcon_feed_status.STATUS_NOT_AVAILABLE,
            'missing_files': ','.join(file_name for file_name in missing_files)
        }
    return {'file_status': garcon_feed_status.STATUS_DOWNLOADED}


@task.decorate(timeout=1000)
def check_files_on_s3(
        activity, feed_name, date, s3_download_path,
        file_pattern, list_output=False):
    """Check if there are some new files in s3_download_path.

    Args:
        activity (ActivityWorker): The activity worker.
        feed_name (str): Feed name of workflow execution for status updates.
        date (str): Reporting date (YYYY-MM-DD).
        s3_download_path (str): Drop location on S3.
        file_pattern (str): File pattern for searching.
        list_output (bool): If True, returns a list of file names instead
            of list of dicts.
    Returns:
        source_files_dict (dict): Dict with files metadata.
        Supported formats of source_files_dict:
             - dict format ( by default):
                {
                    'files': [
                        {'file_name': 'file1', 'found': True, 'file_size': 1},
                        {'file_name': 'file2', 'found': False, 'file_size': 1},
                        {'file_name': 'file3', 'found': True, 'file_size': 1}
                    ]
                }
            - list format ( if list_output is True):
                {
                    'files': [ 'file1.csv', 'file2.csv']
                }
    """
    files_on_s3 = []
    for file_path in s3utils.get_list_of_files_and_directories(
            s3_download_path):
        # print(file_path)
        if re.match(file_pattern, str(file_path)):
            files_on_s3.append(os.path.basename(file_path))

    ingested_files = task_status.get_values(
        feed_name, date, 'ingested_files_status')

    # if there are new files
    if set(files_on_s3) - set(ingested_files):
        activity.logger.info(
            'New files have found for {} {}'.format(feed_name, date))
        if list_output:
            return {
                'source_files_dict': {
                    'files': [file_name for file_name in files_on_s3]
                }
            }
        else:
            return {
                'source_files_dict': {
                    'files': [
                        {'file_name': file_name,
                         'found': True} for file_name in files_on_s3
                    ]}
            }

    if not ingested_files:
        set_overall_status_enhanced(
            feed_name, date, garcon_feed_status.STATUS_NOT_AVAILABLE, activity)
        activity.logger.info(
            'For {} {} there are no any new files'.format(feed_name, date))
    return {'stop': True}


@task.decorate(timeout=600)
def mark_ingested_files(activity, feed_name, date, source_files_dict):
    """Put list of injected files in DynamoDb.

    Args:
        activity (ActivityWorker): The activity worker.
        feed_name (str): Feed name of workflow execution for status updates.
        date (str): Reporting date (YYYY-MM-DD).
        source_files_dict: dict containing metadata of files.
            Supported formats of source_files_dict:
             - dict format:
                {
                    'files': [
                        {'file_name': 'file1', 'found': True, 'file_size': 1},
                        {'file_name': 'file2', 'found': False, 'file_size': 1},
                        {'file_name': 'file3', 'found': True, 'file_size': 1}
                    ]
                }
            - list format:
                {
                    'files': [ 'file1.csv', 'file2.csv']
                }
    """
    files_list = source_files_dict.get('files', [])
    if not files_list:
        files = []
    elif isinstance(files_list[0], dict):
        files = [file['file_name'] for file in files_list if file.get('found')]
    else:
        files = list(files_list)
    task_status.set_values(feed_name, date, 'ingested_files', files)
    activity.logger.info(
        'Mark ingested files for {} {}'.format(feed_name, date))


def get_contexts_config_for_report(
        report_name, contexts_config, licensor=None):
    """Get required/optional contexts for given report.

    Args:
        report_name (str): Name of report. 'default' is applicable to
            all reports.
        contexts_config (dict): configuration for contexts.
        licensor (str): licensor name. Optional. If licensor is not specified,
            then it is applicable to all licensors.

    Either 'required' or 'optional' section should be provided.
    If there is no config, then context is required.
    Format:
     "contexts": {
        "theorchard": {
             "default": {
                 "required": ["US", "UK"],
             },
             "report-1": {
                "optional": ["IT"]
              },
             "report-N": {...}
        }
    }
    """
    if not licensor:
        licensor = 'default'

    licensor_conf = (contexts_config.get(licensor, {})
                     or contexts_config.get('default', {}))
    conf = (licensor_conf.get(report_name, {})
            or licensor_conf.get('default', {}))
    optional = False
    contexts = conf.get('required', [])
    if not contexts and conf.get('optional', []):
        contexts = conf.get('optional', [])
        optional = True
    return optional, contexts


def file_is_optional(report_name, contexts_config, file_name,
                     licensor=None) -> bool:
    """Check if file is optional or required.

    Args:
        report_name (str): name of report, e.g. amContent.
        contexts_config (dict): configuration for optional
            and required contexts.
        file_name (str): name of the file.
        licensor (str): licensor name. Optional.

    Returns:
        bool: True if file is optional.
    """
    if not report_name:
        return False
    if not licensor:
        licensor = 'default'

    optional, contexts = get_contexts_config_for_report(
        report_name, contexts_config, licensor)
    files = [f'.*{context}.*' for context in contexts]

    for of in files:
        if re.match(of, file_name):
            return optional

    # no match
    return not optional


def has_newer_ingested_date(feed_name, date):
    """Return True if the feed has an INGESTED date later than `date`.

    The feed-status table is keyed (feed_name HASH, date RANGE) with `date`
    stored as an ISO 'YYYY-MM-DD' string, so a descending sort-key range query
    locates newer dates directly. Filters on INGESTED, so a failed or in-flight
    date never raises the high-water mark and blocks the dates after it.
    Paginates because the FilterExpression is applied after the key query, so
    an early page can be empty while a later page still matches.

    Args:
        feed_name (str): Feed name.
        date (str): Requested reporting date, 'YYYY-MM-DD'.

    Returns:
        bool: True if a strictly newer INGESTED date exists.
    """
    table = boto3.resource(
        'dynamodb', region_name=feed_status_config.aws_region
    ).Table(feed_status_config.feed_ingestion_table)

    query_kwargs = dict(
        KeyConditionExpression=(
            Key('feed_name').eq(feed_name) & Key('date').gt(date)),
        FilterExpression=Attr('status').eq(
            garcon_feed_status.STATUS_INGESTED),
        ScanIndexForward=False)

    while True:
        response = table.query(**query_kwargs)
        if response['Items']:
            return True
        start_key = response.get('LastEvaluatedKey')
        if not start_key:
            return False
        query_kwargs['ExclusiveStartKey'] = start_key