"""YouTube Channel Names Workflow.""" from garcon.param import StaticParam from garcon_contrib.dynamo_feed_status import garcon_feed_status from feed_ingestion.flows import base from feed_ingestion.flows.youtube_channel_names import config from feed_ingestion.flows.youtube_channel_names import tasks from feed_ingestion.tasks import load_raw_table_tasks_sf from feed_ingestion.tasks import overall_status_tasks class Flow(base.FlowBase, base.FlowConfigMixin): """Class representing the workflow.""" def __init__(self): """Initialize flow object.""" super(Flow, self).__init__(config.feed_name, config.feed_version) def decider(self, schedule): """Activity decider. Args: schedule (callable): The scheduler method. """ bootstrap = schedule('bootstrap', self.bootstrap) if bootstrap.result.get('bootstrap.stop') is True: return get_missing_channels = schedule( 'get_missing_channels', self.get_missing_channels, requires=[bootstrap]) create_temp_staging_raw_table = schedule( 'create_temp_staging_raw_table', self.create_temp_staging_raw_table, requires=[get_missing_channels]) load_temp_staging_raw_table = schedule( 'load_temp_staging_raw_table', self.load_temp_staging_raw_table, requires=[create_temp_staging_raw_table]) update_dim_table = schedule( 'update_dim_table', self.update_dim_table, requires=[load_temp_staging_raw_table]) drop_temp_staging_table = schedule( 'drop_temp_staging_table', self.drop_temp_staging_table, requires=[update_dim_table]) schedule( 'set_status_to_ingested', self.set_status_to_ingested, requires=[drop_temp_staging_table]) def contextified_feed_name(self, context): """Get feed_name in context. Args: context (dict): The context of the flow. Returns: str: Contextified feed name. """ assert 'licensor' in context, 'There is no licensor in context' licensor = context['licensor'] assert licensor in config.licensors, \ 'There is no such licensor in config file' return '_'.join([self.feed_name, licensor]) def workflow_id(self, initial_context): """Generate workflow id. Args: initial_context (dict): The initial context for the flow. Returns: str: A unique identifier for a workflow being executed. """ licensor = initial_context['licensor'] date = initial_context['context_date'] return '{flow_name}-{licensor}-{date}'.format( flow_name=self.name, licensor=licensor, date=date) @property def bootstrap(self): """Bootstrap initial configuration.""" return self.create( name='bootstrap', tasks=base.SyncRunner( tasks.bootstrap.fill( feed_name=StaticParam(config.feed_name), namespace='bootstrap', date='context_date', reload='reload', licensor='licensor'))) @property def get_missing_channels(self): """Get missing channel names and upload to S3.""" return self.create( name='get_missing_channel_ids', tasks=base.SyncRunner( tasks.get_missing_channels.fill( namespace='get_missing_channel_ids', feed_name='bootstrap.feed_name', date='bootstrap.date', processed_filename='bootstrap.processed_filename', s3_preprocessed_path='bootstrap.s3_preprocessed_path', credentials_path='bootstrap.credentials_path'))) @property def update_dim_table(self): """Update channel names mapping table.""" return self.create( name='update_dim_table', tasks=base.SyncRunner( tasks.update_dim_table.fill( namespace='update_dim_table', feed_name='bootstrap.feed_name', date='bootstrap.date', temp_staging_raw_table='bootstrap.temp_staging_raw_table' ))) @property def create_temp_staging_raw_table(self): """Create temp staging raw table.""" return self.create( name='create_temp_staging_raw_table', schedule_to_start=48000, tasks=base.SyncRunner( load_raw_table_tasks_sf.create_temp_staging_raw_table.fill( namespace='create_temp_staging_raw_table', date='bootstrap.date', sfdb_params='sfdb_params', feed_name='bootstrap.feed_name', secrets_path=StaticParam(config.secrets_path), temp_staging_raw_table='bootstrap.temp_staging_raw_table' ) ) ) @property def load_temp_staging_raw_table(self): """Load temp staging raw table.""" return self.create( name='load_temp_staging_raw_table', schedule_to_start=48000, tasks=base.SyncRunner( load_raw_table_tasks_sf.load_temp_staging_raw_table.fill( namespace='load_temp_staging_raw_table', date='bootstrap.date', sfdb_params='sfdb_params', feed_name='bootstrap.feed_name', secrets_path=StaticParam(config.secrets_path), kwargs='bootstrap.temp_staging_table_kwargs', key_dir='bootstrap.s3_preprocessed_path', temp_staging_raw_table='bootstrap.temp_staging_raw_table' ) ) ) @property def drop_temp_staging_table(self): """Drop the temporary staging table.""" return self.create( name='drop_temp_staging_table', schedule_to_start=48000, tasks=base.SyncRunner( load_raw_table_tasks_sf.drop_temp_staging_raw_table.fill( namespace='drop_temp_staging_table', feed_name='bootstrap.feed_name', sfdb_params='sfdb_params', secrets_path=StaticParam(config.secrets_path), temp_staging_raw_table='bootstrap.temp_staging_raw_table' ) ) ) @property def set_status_to_ingested(self): """Set overall feed status to INGESTED.""" return self.create( name='set_status_to_ingested', schedule_to_start=48000, tasks=base.SyncRunner( overall_status_tasks.set_overall_status.fill( namespace='set_status_to_ingested', feed_name='bootstrap.feed_name', date='bootstrap.date', status=StaticParam( garcon_feed_status.STATUS_INGESTED))))