"""Spotify Ingestion Workflow."""

from garcon import param
from garcon.param import StaticParam
from garcon_contrib.dynamo_feed_status import garcon_feed_status

from feed_ingestion.flows import base
from feed_ingestion.flows.spotify import config
from feed_ingestion.flows.spotify import tasks
from feed_ingestion.tasks import load_fact_tables_tasks_sf
from feed_ingestion.tasks import load_raw_table_tasks_sf
from feed_ingestion.tasks import overall_status_tasks
from feed_ingestion.tasks import validate_raw_data_tasks_sf
from feed_ingestion.util.jenkins.tasks import build_jenkins_dbt


class Flow(
        base.FlowLicensor, base.FlowConfigMixin,
        base.FlowLoadRawMixinSF):
    """Class representing the workflow."""

    def __init__(self):
        """Initialize flow object."""
        super(Flow, self).__init__(config.feed_name, config.feed_version)
        self.timeout = 60 * 60 * 2  # override default to 2 hours

    def decider(self, schedule):
        """Activity decider.

        Args:
            schedule (callable): The scheduler method.
        """
        check_date = schedule(
            'check_date', self.check_date)

        if check_date.result.get('check_date.stop') is True:
            return

        check_feed_status = schedule(
            'check_feed_status',
            self.check_feed_status,
            requires=[check_date])
        if check_feed_status.result.get('check_feed_status.stop') is True:
            return

        bootstrap = schedule(
            'bootstrap',
            self.bootstrap,
            requires=[check_feed_status])
        if bootstrap.result.get('bootstrap.stop') is True:
            return

        if bootstrap.result.get('bootstrap.use_s3') == 'True':
            if bootstrap.result.get('bootstrap.licensor') \
                    in ('sme', 'smejp', 'smejpintl'):
                grab_drop_files_from_s3 = schedule(
                    'grab_drop_files_from_s3',
                    self.grab_drop_files_from_s3,
                    requires=[bootstrap])
                last_activities = [grab_drop_files_from_s3]
        else:
            if bootstrap.result.get('bootstrap.use_partitioned') == 'True':
                grab_drop_files_partitioned = schedule(
                    'grab_drop_files_partitioned',
                    self.grab_drop_files_partitioned,
                    requires=[bootstrap])
                if grab_drop_files_partitioned.result.get(
                        'grab_drop_files.stop') is True:
                    return
                last_activities = [grab_drop_files_partitioned]
            else:
                grab_drop_files = schedule(
                    'grab_drop_files',
                    self.grab_drop_files,
                    requires=[bootstrap])
                last_activities = [grab_drop_files]

        check_available_reports = schedule(
            'check_available_reports',
            self.check_available_reports,
            requires=last_activities)

        if check_available_reports.result.get(
                'check_available_reports.stop') is True:
            return

        # Get dict with available activities.
        # Possible activities are - load_common_table,
        # load_staging_raw_table, load_staging_fact_table.
        necessary_activities = check_available_reports.result.get(
            'check_available_reports.necessary_activities')

        create_temp_staging_raw_table = schedule(
            'create_temp_staging_raw_table',
            self.create_temp_staging_raw_table,
            requires=[check_available_reports])

        load_temp_staging_raw_table = schedule(
            'load_temp_staging_raw_table',
            self.load_temp_staging_raw_table,
            requires=[create_temp_staging_raw_table])

        last_activities = [load_temp_staging_raw_table]

        if necessary_activities.get('load_common_table', False):
            create_transitional_common_tables = schedule(
                'create_transitional_common_tables',
                self.create_transitional_common_tables,
                requires=[load_temp_staging_raw_table])

            load_common_tables = schedule(
                'load_transitional_common_tables',
                self.load_common_tables,
                requires=[create_transitional_common_tables])

            last_activities = [load_common_tables]

        if necessary_activities.get('load_staging_raw', False):
            load_staging_raw_table = schedule(
                'load_staging_raw_table',
                self.load_staging_raw_table,
                requires=last_activities)

            last_activities = [load_staging_raw_table]

        drop_temp_staging_table = schedule(
            'drop_temp_staging_table',
            self.drop_temp_staging_table,
            requires=last_activities)
        last_activities = [drop_temp_staging_table]

        if necessary_activities.get('load_fact_analytics', False):
            update_dim_tables = schedule(
                'update_dim_tables',
                self.update_dim_tables,
                requires=last_activities)

            load_staging_fact_table = schedule(
                'load_staging_fact_table',
                self.load_staging_fact_table,
                requires=[update_dim_tables])

            # load fact data into Snowflake
            load_fact_tables = schedule(
                'load_fact_tables',
                self.load_fact_tables,
                requires=[load_staging_fact_table])

            last_activities = [load_fact_tables]

        load_aggregated_skips_and_saves = schedule(
            'load_aggregated_skips_and_saves',
            self.load_aggregated_skips_and_saves,
            requires=last_activities)

        set_status_ingested = schedule(
            'set_status_ingested',
            self.set_status_ingested,
            requires=[load_aggregated_skips_and_saves])

        build_jenkins_dbt = schedule(
            'build_jenkins_dbt',
            self.build_jenkins_dbt,
            requires=[set_status_ingested])

        schedule(
            'set_overall_status_ingested',
            self.set_overall_status_ingested,
            requires=[build_jenkins_dbt])

    @property
    def check_date(self):
        """Check if it's possible to ingest a date."""
        return self.create(
            name='check_date',
            tasks=base.SyncRunner(
                tasks.check_date.fill(
                    namespace='check_date',
                    date='context_date',
                    licensor='licensor',
                    use_s3='use_s3')))

    @property
    def check_feed_status(self):
        """Check and reset feed status if it is needed."""
        return self.create(
            name='check_feed_status',
            tasks=base.SyncRunner(
                tasks.check_feed_status.fill(
                    namespace='check_feed_status',
                    date='context_date',
                    reload='reload',
                    licensor='licensor')))

    @property
    def bootstrap(self):
        """Bootstrap initial configuration."""
        return self.create(
            name='bootstrap',
            tasks=base.SyncRunner(
                tasks.bootstrap.fill(
                    namespace='bootstrap',
                    date='context_date',
                    reload='reload',
                    reports='reports',
                    use_partitioned='use_partitioned',
                    licensor='licensor',
                    use_s3='use_s3'
                )))

    @property
    def grab_drop_files(self):
        """Request Spotify API and archive files on s3."""
        return self.create(
            name='grab_drop_files',
            generators=[self.grab_drop_files_generator],
            schedule_to_start=48000,
            tasks=base.SyncRunner(
                tasks.grab_drop_files.fill(
                    namespace='grab_drop_files',
                    feed_name='feed_name',
                    date='bootstrap.date',
                    report_name='report_name',
                    archive_path='archive_path',
                    licensor='licensor')))

    @property
    def grab_drop_files_partitioned(self):
        """Request Spotify API and archive files on s3."""
        return self.create(
            name='grab_drop_files_partitioned',
            generators=[self.grab_drop_files_generator],
            schedule_to_start=48000,
            tasks=base.SyncRunner(
                tasks.grab_drop_files_partitioned.fill(
                    namespace='grab_drop_files',
                    feed_name='feed_name',
                    date='bootstrap.date',
                    report_name='report_name',
                    archive_path='archive_path',
                    licensor='licensor')))

    @property
    def grab_drop_files_from_s3(self):
        """Copy files from drop location and archive files on s3."""
        return self.create(
            name='grab_drop_files_from_s3',
            generators=[self.grab_drop_files_generator],
            schedule_to_start=48000,
            tasks=base.SyncRunner(
                tasks.grab_drop_files_from_s3.fill(
                    namespace='grab_drop_files_from_s3',
                    feed_name='feed_name',
                    date='bootstrap.date',
                    report_name='report_name',
                    archive_path='archive_path',
                    drop_path='drop_path',
                    licensor='licensor')))

    @property
    def check_available_reports(self):
        """Check reports which was uploaded on s3."""
        return self.create(
            name='check_available_reports',
            schedule_to_start=48000,
            tasks=base.SyncRunner(
                tasks.check_available_reports.fill(
                    namespace='check_available_reports',
                    date='bootstrap.date',
                    reports_status_names='bootstrap.reports_status_names',
                    licensor='licensor')))

    @property
    def create_temp_staging_raw_table(self):
        """Create temp staging raw table."""
        return self.create(
            name='create_temp_staging_raw_table',
            generators=[self.temp_staging_tables_generator],
            schedule_to_start=48000,
            tasks=base.SyncRunner(
                load_raw_table_tasks_sf.create_temp_staging_raw_table.fill(
                    namespace='create_temp_staging_raw_table',
                    date='bootstrap.date',
                    sfdb_params='sfdb_params',
                    feed_name='feed_name',
                    secrets_path=StaticParam(config.secrets_path),
                    temp_staging_raw_table='temp_staging_raw_table')))

    @property
    def validate_raw_data(self):
        """Validate data on s3.

        Validates source files, logs data errors via Sentry
        and aborts execution if error rate is too high.
        """
        return self.create(
            name='validate_raw_data',
            generators=[self.temp_staging_tables_generator],
            schedule_to_start=48000,
            tasks=base.SyncRunner(
                validate_raw_data_tasks_sf.validate_raw_data.fill(
                    namespace='validate_raw_data',
                    feed_name='feed_name',
                    date='bootstrap.date',
                    sfdb_params='sfdb_params',
                    key_dir='key_dir',
                    temp_staging_raw_table='temp_staging_raw_table',
                    error_limit=(
                        param.StaticParam(config.snowflake_error_limit)),
                    kwargs='kwargs')))

    @property
    def load_temp_staging_raw_table(self):
        """Load temp staging raw table."""
        return self.create(
            name='load_temp_staging_raw_table',
            generators=[self.temp_staging_tables_generator],
            schedule_to_start=48000,
            tasks=base.SyncRunner(
                validate_raw_data_tasks_sf.load_temp_staging_raw_table.fill(
                    namespace='load_temp_staging_raw_table',
                    date='bootstrap.date',
                    sfdb_params='sfdb_params',
                    feed_name='feed_name',
                    secrets_path=StaticParam(config.secrets_path),
                    kwargs='kwargs',
                    key_dir='key_dir',
                    temp_staging_raw_table='temp_staging_raw_table',
                    error_limit=param.StaticParam(
                        config.snowflake_error_limit))))

    @property
    def create_transitional_common_tables(self):
        """Create transitional tables for common report."""
        return self.create(
            name='create_transitional_common_tables',
            generators=[self.transitional_common_tables_generator],
            schedule_to_start=48000,
            tasks=base.SyncRunner(
                tasks.create_transitional_common_tables.fill(
                    namespace='create_transitional_common_tables',
                    date='bootstrap.date',
                    sfdb_params='sfdb_params',
                    feed_name='feed_name',
                    transitional_temp_table='transitional_temp_table',
                    report_name='report_name')))

    @property
    def load_common_tables(self):
        """Activity to load spotify_users and spotify_tracks tables."""
        return self.create(
            name='load_common_tables',
            generators=[self.transitional_common_tables_generator],
            schedule_to_start=48000,
            tasks=base.SyncRunner(
                tasks.load_common_tables.fill(
                    namespace='load_common_tables',
                    date='bootstrap.date',
                    feed_name='feed_name',
                    report_name='report_name',
                    sfdb_params='sfdb_params',
                    temp_staging_raw_table='temp_staging_raw_table',
                    transitional_temp_table='transitional_temp_table',
                    staging_raw_table='staging_raw_table',
                    licensor='licensor'),
                overall_status_tasks.set_overall_status.fill(
                    date='bootstrap.date',
                    feed_name='feed_name',
                    set_status_once=param.StaticParam(True),
                    status=param.StaticParam(
                        garcon_feed_status.STATUS_POPULATED_RAW_TABLE))))

    @property
    def load_staging_raw_table(self):
        """Load permanent staging raw table."""
        activity_name = 'load_staging_raw_table'
        return self.create(
            name=activity_name,
            generators=[self.load_staging_raw_table_generator],
            schedule_to_start=48000,
            tasks=base.SyncRunner(
                load_raw_table_tasks_sf.load_staging_raw_table.fill(
                    namespace=activity_name,
                    date='bootstrap.date',
                    feed_name='feed_name',
                    secrets_path=StaticParam(config.secrets_path),
                    sfdb_params='sfdb_params',
                    staging_raw_table='staging_raw_table',
                    kwargs='kwargs'),
                overall_status_tasks.set_overall_status.fill(
                    date='bootstrap.date',
                    feed_name='feed_name',
                    set_status_once=param.StaticParam(True),
                    status=param.StaticParam(
                        garcon_feed_status.STATUS_POPULATED_RAW_TABLE))))

    @property
    def load_aggregated_skips_and_saves(self):
        """Activity to load load_aggregated_skips_and_saves."""
        return self.create(
            name='load_aggregated_skips_and_saves',
            tasks=base.SyncRunner(
                tasks.load_aggregated_skips_and_saves.fill(
                    namespace='load_aggregated_skips_and_saves',
                    date='bootstrap.date',
                    feed_name='bootstrap.aggregated_feed_name',
                    sfdb_params='sfdb_params',
                    licensor='licensor')))

    @property
    def drop_temp_staging_table(self):
        """Drop the temporary staging tables."""
        return self.create(
            name='drop_temp_staging_table',
            generators=[self.temp_staging_tables_generator],
            schedule_to_start=48000,
            tasks=base.SyncRunner(
                tasks.drop_temp_table.fill(
                    sfdb_params='sfdb_params',
                    namespace='drop_temp_staging_table',
                    temp_table_name='temp_staging_raw_table')))

    @property
    def update_dim_tables(self):
        """Update dimension tables (dim_user and dim_playlist)."""
        return self.create(
            name='update_dim_tables',
            schedule_to_start=48000,
            tasks=base.SyncRunner(
                load_fact_tables_tasks_sf.update_dim_tables.fill(
                    namespace='update_dim_tables',
                    feed_name='bootstrap.facts_feed_name',
                    secrets_path=StaticParam(config.secrets_path),
                    date='bootstrap.date',
                    sfdb_params='sfdb_params',
                    kwargs='bootstrap.dimension_tables'),
                tasks.sns_publish_message.fill(
                    topic=param.StaticParam(
                        config.dimension_tables['sns_topic']),
                    feed_name='bootstrap.facts_feed_name',
                    date='bootstrap.date',
                    subject='update_dim_tables.sns_report_subject',
                    message='update_dim_tables.sns_report_message')))

    @property
    def load_staging_fact_table(self):
        """Activity to load staging_fact_analytics_{feed_name}_{datestamp}."""
        return self.create(
            name='unload_fact_data',
            schedule_to_start=48000,
            tasks=base.SyncRunner(
                load_fact_tables_tasks_sf.create_staging_fact.fill(
                    namespace='create_staging_fact',
                    feed_name='bootstrap.facts_feed_name',
                    secrets_path=StaticParam(config.secrets_path),
                    date='bootstrap.date',
                    sfdb_params='sfdb_params'),
                load_fact_tables_tasks_sf.load_staging_fact.fill(
                    namespace='load_staging_fact',
                    feed_name='bootstrap.facts_feed_name',
                    secrets_path=StaticParam(config.secrets_path),
                    date='bootstrap.date',
                    sfdb_params='sfdb_params',
                    kwargs='kwargs')))

    @property
    def load_fact_tables(self):
        """Activity to load fact_analytics & fact_analytics_error.

        Deletes any existing data for this feed and date in fact_analytics &
        fact_analytics_error.

        After successfully loading the data, sets the feed's status for that
        download_date to INGESTED.
        """
        activity_name = 'load_fact_tables'

        return self.create(
            name=activity_name,
            schedule_to_start=48000,
            tasks=base.SyncRunner(
                load_fact_tables_tasks_sf.load_fact_data.fill(
                    namespace='load_fact_data',
                    feed_name='bootstrap.facts_feed_name',
                    secrets_path=StaticParam(config.secrets_path),
                    date='bootstrap.date',
                    sfdb_params='sfdb_params',
                    kwargs='kwargs'),
                overall_status_tasks.set_overall_status.fill(
                    feed_name='bootstrap.facts_feed_name',
                    date='bootstrap.date',
                    set_status_once=param.StaticParam(True),
                    status=param.StaticParam(
                        garcon_feed_status.STATUS_INGESTED))))

    @property
    def set_status_ingested(self):
        """Set report status INGESTED if all tasks completed."""
        return self.create(
            name='set_status_ingested',
            schedule_to_start=48000,
            tasks=base.SyncRunner(
                tasks.set_status_ingested.fill(
                    namespace='set_status_ingested',
                    date='bootstrap.date',
                    reports_status_names='bootstrap.reports_status_names')))

    @property
    def build_jenkins_dbt(self):
        """Build Jenkins DBT job."""
        return self.create(
            name='build_jenkins_dbt',
            schedule_to_start=48000,
            tasks=base.SyncRunner(
                build_jenkins_dbt.fill(
                    namespace='build_jenkins_dbt',
                    date='bootstrap.date',
                    feed_name=StaticParam(config.feed_name),
                    licensor='bootstrap.licensor',
                    build_dbt='build_dbt',
                    # coming from context of the https://scheduler.theorchard.io/job/swf-spotify-exec/  # noqa
                    config='bootstrap.jenkins_config')))

    @property
    def set_overall_status_ingested(self):
        """Set overall feed status INGESTED if all reports are completed."""
        return self.create(
            name='set_overall_status_ingested',
            schedule_to_start=48000,
            tasks=base.SyncRunner(
                tasks.set_overall_status_ingested.fill(
                    namespace='set_overall_status_ingested',
                    date='bootstrap.date',
                    licensor='licensor')))

    def grab_drop_files_generator(self, context):
        """Generate parameters for grab_drop_files activity.

        Args:
            context (dict): The current context.
        Yields:
            dict: Dictionary with report_name and feed name,
                which will be used for writing statuses in DynamoDB.
        """
        reports_status_names = context['bootstrap.reports_status_names']
        archive_paths = context['bootstrap.archive_paths']
        drop_paths = context['bootstrap.drop_paths']
        for report_name, feed_name in reports_status_names.items():
            yield dict(
                report_name=report_name,
                feed_name=feed_name,
                archive_path=archive_paths[report_name],
                drop_path=drop_paths[report_name])

    def temp_staging_tables_generator(self, context):
        """Generate parameters for temporary staging tables.

        Used by the create_temp_staging_raw_table,
        load_temp_staging_raw_table activity and drop_temp_staging_table.

        Args:
            context (dict): The current context.
        Yields:
            dict: Dictionary of temporary staging table names.
        """
        file_pattern = '.*.gz'
        reports_status_names = context[
            'check_available_reports.available_reports']
        archive_paths = context['bootstrap.archive_paths']
        temp_staging_raw_names = context['bootstrap.temp_staging_raw_names']
        licensor = context['bootstrap.licensor']
        for report_name, feed_name in reports_status_names.items():
            key_dir = 's3://{bucket}/{dir}'.format(
                bucket=config.data_bucket,
                dir=archive_paths[report_name])
            yield dict(
                temp_staging_raw_table=temp_staging_raw_names[report_name],
                feed_name=feed_name,
                key_dir=key_dir,
                kwargs=dict(
                    file_pattern=file_pattern.format(
                        file_type=report_name,
                        licensor=licensor)))

    def load_staging_raw_table_generator(self, context):
        """Generate parameters for load_staging_raw_table.

        Used by the load_staging_raw_table.

        Args:
            context (dict): The current context.
        Yields:
            dict: Dictionary of temporary staging table names,
                staging_raw_table.
        """
        available_reports = context[
            'check_available_reports.available_reports']
        licensor = context['bootstrap.licensor']
        for report_name, feed_name in available_reports.items():
            if report_name not in config.common_reports:
                yield dict(
                    staging_raw_table=(
                        config.reports[report_name]['staging_raw']),
                    feed_name=feed_name,
                    kwargs=dict(
                        temp_staging_raw_names=context[
                            'bootstrap.temp_staging_raw_names'],
                        report_name=report_name,
                        query='load_staging_raw_{report_name}'.format(
                            report_name=report_name),
                        licensor=licensor))

    def transitional_common_tables_generator(self, context):
        """Generate parameters for loading common tables.

        Used by the create_transitional_common_tables and load_common_tables.

        Args:
            context (dict): The current context.
        Yields:
            dict: Dictionary of temporary staging table names,
                staging_raw_table.
        """
        available_reports = context[
            'check_available_reports.available_reports']
        temp_staging_raw_names = context['bootstrap.temp_staging_raw_names']
        licensor = context['bootstrap.licensor']
        for report_name in available_reports.keys() & set(
                config.common_reports):
            if 'staging_raw' in config.reports[report_name]:
                transitional_temp_table = (
                    config.transitional_temp_table.format(
                        date=context['bootstrap.date'].replace('-', ''),
                        report=report_name,
                        licensor=licensor))
                yield dict(
                    report_name=report_name,
                    feed_name=available_reports[report_name],
                    transitional_temp_table=transitional_temp_table,
                    staging_raw_table=config.reports[
                        report_name]['staging_raw'],
                    temp_staging_raw_table=temp_staging_raw_names[report_name],
                    licensor=licensor)