"""Amazon DataPulse Ingestion Workflow.""" from garcon.param import StaticParam from garcon_contrib.dynamo_feed_status import garcon_feed_status from snowflake_connector.etl_connector import SQLLoader from feed_ingestion.common.base_executor import SnowflakeTemplatedSQLExecutor from feed_ingestion.flows import base from feed_ingestion.flows.amazon_datapulse import config from feed_ingestion.flows.amazon_datapulse import tasks from feed_ingestion.flows.amazon_datapulse.stage_loader import ( AmazonDataPulseSL, ) from feed_ingestion.tasks import overall_status_tasks sql_loader = SQLLoader(__file__) class Flow(base.FlowBase, base.FlowConfigMixin): """Amazon DataPulse workflow class.""" def __init__(self): """Initialize flow object.""" super().__init__( feed_name=config.feed_name, version=config.feed_version, ) self.timeout = 60 * 60 * 3 def workflow_id(self, initial_context): """Generate a unique workflow id. Args: initial_context (dict): The initial context for the flow. Returns: str: A unique identifier for a workflow being executed. """ date = initial_context['context_date'] feed_name = self.contextified_feed_name(initial_context) return f'{feed_name}-{date}' def contextified_feed_name(self, context): """Get feed_name in context. Args: context (dict): The context of the flow. Returns: str: Contextified feed name. """ report_name = context['report_name'] feed_name = config.get_contextified_feed_name( report_name=report_name, partition=context.get('partition'), ) return feed_name def decider(self, schedule): """Define the workflow activity DAG. Args: schedule (callable): The scheduler method. """ bootstrap = schedule('bootstrap', self.bootstrap) if bootstrap.result.get('bootstrap.stop'): return fetch_from_athena = schedule( 'fetch_from_athena', self.fetch_from_athena, requires=[bootstrap] ) load_staging_raw_table = schedule( 'load_staging_raw_table', self.load_staging_raw_table, requires=[fetch_from_athena], ) schedule( 'set_status_to_ingested', self.set_status_to_ingested, requires=[load_staging_raw_table], ) @property def bootstrap(self): """Bootstrap initial configuration.""" return self.create( name='bootstrap', tasks=base.SyncRunner( tasks.bootstrap.fill( namespace='bootstrap', date='context_date', reload='reload', report_name='report_name', partition='partition', ) ), ) @property def fetch_from_athena(self): """Fetch report data from Athena and write PARQUET files to S3.""" return self.create( name='fetch_from_athena', tasks=base.SyncRunner( tasks.fetch_from_athena.fill( namespace='fetch_from_athena', feed_name='bootstrap.feed_name', report_name='bootstrap.report_name', athena_source_table='bootstrap.athena_source_table', date='bootstrap.date', destination_s3_bucket='bootstrap.archive_bucket', destination_s3_path='bootstrap.archive_path', partition='bootstrap.partition', ) ), ) load_staging_raw_table = AmazonDataPulseSL.load_activity( secrets_path=config.secrets_path, sql_loader=sql_loader, executor_class=SnowflakeTemplatedSQLExecutor, requirements=dict( date='bootstrap.date', feed_name='bootstrap.feed_name', report='bootstrap.report_name', staging_raw_table_name='bootstrap.staging_raw_table', s3_dir_path='bootstrap.s3_dir_path', kwargs='bootstrap.kwargs', ), ) @property def set_status_to_ingested(self): """Set feed status to STATUS_INGESTED.""" return self.create( name='set_status_to_ingested', tasks=base.SyncRunner( overall_status_tasks.set_overall_status.fill( feed_name='bootstrap.feed_name', date='bootstrap.date', status=StaticParam(garcon_feed_status.STATUS_INGESTED), ) ), )