"""YouTube Video Report Workflow.""" from garcon import param from garcon.param import StaticParam from garcon_contrib.dynamo_feed_status import garcon_feed_status from snowflake_connector.etl_connector import SQLLoader from feed_ingestion.flows import base from feed_ingestion.flows.youtube_video import config from feed_ingestion.flows.youtube_video import tasks from feed_ingestion.flows.youtube_video.stage_loader import YouTubeVideoSL from feed_ingestion.tasks import overall_status_tasks, s3_tasks, youtube_tasks class Flow(base.FlowBase, base.FlowConfigMixin, base.FlowLoadRawMixinSF, base.FlowYouTubeMixin): """Class representing the workflow.""" def __init__(self): """Initialize flow object.""" super(Flow, self).__init__(config.feed_name, config.feed_version) def decider(self, schedule, context): """Activity decider. Args: schedule (callable): The scheduler method. """ bootstrap = schedule('bootstrap', self.bootstrap) if bootstrap.result.get('bootstrap.stop') is True: return licensor = bootstrap.result.get('bootstrap.licensor') if licensor in ['theorchard']: if context.get('skip_grab_reports_files') == 'True': next_task_requires = bootstrap else: grab_drop_files = schedule( 'grab_reports_files', self.grab_reports_files, requires=[bootstrap]) next_task_requires = grab_drop_files if grab_drop_files.result.get('grab_reports_files.stop'): return else: assert licensor == 'sme' sme_copy_from_athena_to_s3 = schedule( 'sme_copy_from_athena_to_s3', self.sme_copy_from_athena_to_s3, requires=[bootstrap]) next_task_requires = sme_copy_from_athena_to_s3 if sme_copy_from_athena_to_s3.result.get( 'sme_copy_from_athena_to_s3.stop'): return source_files = schedule( 'source_files', self.source_files, requires=[next_task_requires]) create_temp_staging_raw_table = schedule( 'create_temp_staging_raw_table', self.create_temp_staging_raw_table, requires=[source_files]) load_temp_staging_raw_table = schedule( 'load_temp_staging_raw_table', self.load_temp_staging_raw_table, requires=[create_temp_staging_raw_table]) clean_staging_raw_table = schedule( 'clean_staging_raw_table', self.clean_staging_raw_table, requires=[load_temp_staging_raw_table]) load_staging_raw_table = schedule( 'load_staging_raw_table', self.load_staging_raw_table, requires=[clean_staging_raw_table]) drop_temp_staging_raw_table = schedule( 'drop_temp_staging_raw_table', self.drop_temp_staging_raw_table, requires=[load_staging_raw_table]) update_staging_raw_table = schedule( 'update_staging_raw_table', self.update_staging_raw_table, requires=[drop_temp_staging_raw_table]) update_channel_names_table = schedule( 'update_channel_names_table', self.update_channel_names_table, requires=[update_staging_raw_table]) update_mnc_and_owner_columns = schedule( 'update_mnc_and_owner_columns', self.update_mnc_and_owner_columns, requires=[update_channel_names_table]) schedule( 'set_status_to_ingested', self.set_status_to_ingested, requires=[update_mnc_and_owner_columns]) def contextified_feed_name(self, context): """Get feed_name in context. Args: context (dict): The context of the flow. Returns: str: Contextified feed name. """ assert 'licensor' in context, 'There is no licensor in context' licensor = context['licensor'] assert licensor in config.licensors, \ 'There is no such licensor in config file' return '_'.join([self.feed_name, licensor]) def workflow_id(self, initial_context): """Generate workflow id. Args: initial_context (dict): The initial context for the flow. Returns: str: A unique identifier for a workflow being executed. """ licensor = initial_context['licensor'] date = initial_context['context_date'] return '{flow_name}-{licensor}-{date}'.format( flow_name=self.name, licensor=licensor, date=date) @property def bootstrap(self): """Bootstrap initial configuration.""" return self.create( name='bootstrap', tasks=base.SyncRunner( tasks.bootstrap.fill( namespace='bootstrap', date='context_date', reload='reload', licensor='licensor', skip_corrupted_rows='skip_corrupted_rows' ))) @property def source_files(self): """Get list of source files.""" return self.create( name='source_files', tasks=base.SyncRunner( s3_tasks.source_files.fill( namespace='source_files', s3_bucket='bootstrap.s3_bucket', s3_path='bootstrap.source_files_path', file_pattern='bootstrap.source_files_pattern'))) @property def sme_copy_from_athena_to_s3(self): """Bootstrap initial configuration.""" return self.create( name='sme_copy_from_athena_to_s3', tasks=base.SyncRunner( youtube_tasks.sme_copy_from_athena_to_s3.fill( namespace='sme_copy_from_athena_to_s3', date='bootstrap.date', report_status_name='bootstrap.feed_name', sme_athena_database=param.StaticParam( config.sme_athena_database), sme_athena_temp_database=param.StaticParam( config.sme_athena_temp_database), sme_athena_source_table=param.StaticParam( config.youtube_report_full_name), destination_s3_bucket='bootstrap.s3_bucket', destination_s3_path='bootstrap.s3_archive_path', athena_workgroup=param.StaticParam( config.athena_workgroup), aws_region=param.StaticParam(config.athena_aws_region), ))) @property def grab_reports_files(self): """Archive report files to archive location.""" return self.create( name='grab_reports_files', schedule_to_start=48000, tasks=base.SyncRunner( youtube_tasks.grab_reports_files.fill( namespace='grab_reports_files', report_name='bootstrap.report_name', report_status_name='bootstrap.feed_name', date='bootstrap.date', archive_path='bootstrap.s3_dir_path', credentials_path='bootstrap.credentials_path', api_service_name=StaticParam( config.youtube_reporting_api_service_name), api_version=StaticParam( config.youtube_reporting_api_version), jobs_meta_path=StaticParam(config.jobs_meta_path), cms_dict='bootstrap.cms_dict', gz=param.StaticParam(True)))) load_temp_staging_raw_table = YouTubeVideoSL.load_activity( feed_name=config.feed_name, secrets_path=config.secrets_path, sql_loader=SQLLoader(__file__), requirements=dict( date='bootstrap.date', source_files_dict='source_files.source_files_dict', s3_dir_path='bootstrap.s3_dir_path', feed_name='bootstrap.feed_name', temp_staging_raw_table_name='bootstrap.temp_staging_raw_table', licensor='bootstrap.licensor', skip_corrupted_rows='bootstrap.skip_corrupted_rows' )) @property def clean_staging_raw_table(self): """Activity to clean the staging raw table.""" return self.create( name='clean_staging_raw_table', tasks=base.SyncRunner( tasks.clean_staging_raw_table.fill( namespace='update_staging_raw_table', feed_name='bootstrap.feed_name', sfdb_params='sfdb_params', licensor='licensor'))) @property def load_staging_raw_table(self): """Activity to load the data into the staging raw table.""" return self.create( name='load_staging_raw_table', tasks=base.SyncRunner( tasks.load_staging_raw_table.fill( namespace='update_staging_raw_table', feed_name='bootstrap.feed_name', sfdb_params='sfdb_params', temp_staging_raw_table='bootstrap.temp_staging_raw_table' ''))) @property def drop_temp_staging_raw_table(self): """Drop the temporary staging tables.""" return self.create( name='drop_temp_staging_raw_table', tasks=base.SyncRunner( tasks.drop_temp_table.fill( namespace='drop_temp_staging_raw_table', temp_staging_raw_table='bootstrap.temp_staging_raw_table' ''))) @property def update_staging_raw_table(self): """Activity to update isrc in staging raw table.""" return self.create( name='update_channel_names_table', tasks=base.SyncRunner( tasks.update_staging_raw_table.fill( namespace='update_staging_raw_table', date='bootstrap.date', feed_name='bootstrap.feed_name', sfdb_params='sfdb_params', licensor='licensor'))) @property def update_mnc_and_owner_columns(self): """Activity to update MCN and OWNER columns in names mapping table.""" return self.create( name='update_mnc_and_owner_columns', tasks=base.SyncRunner( tasks.update_mnc_and_owner_columns.fill( namespace='update_mnc_and_owner_columns', date='bootstrap.date', feed_name='bootstrap.feed_name', sfdb_params='sfdb_params', channel_names_table_name='bootstrap.channel_names_table', licensor='licensor'))) @property def set_status_to_ingested(self): """Set overall feed status to INGESTED.""" return self.create( name='set_status_to_ingested', schedule_to_start=48000, tasks=base.SyncRunner( overall_status_tasks.set_overall_status.fill( namespace='set_status_to_ingested', feed_name='bootstrap.feed_name', date='bootstrap.date', status=param.StaticParam( garcon_feed_status.STATUS_INGESTED))))