"""YouTube Asset Report Workflow.""" from garcon import param from garcon.param import StaticParam from garcon_contrib.dynamo_feed_status import garcon_feed_status from snowflake_connector.etl_connector import SQLLoader from feed_ingestion.flows import base from feed_ingestion.flows.youtube_asset import config from feed_ingestion.flows.youtube_asset import tasks from feed_ingestion.flows.youtube_asset.stage_loader import YouTubeAssetSL from feed_ingestion.tasks import overall_status_tasks from feed_ingestion.tasks import youtube_tasks class Flow(base.FlowBase, base.FlowConfigMixin): """Class representing the workflow.""" def __init__(self): """Initialize flow object.""" super(Flow, self).__init__(config.feed_name, config.feed_version) def decider(self, schedule): """Activity decider. Args: schedule (callable): The scheduler method. """ bootstrap = schedule('bootstrap', self.bootstrap) if bootstrap.result.get('bootstrap.stop') is True: return licensor = bootstrap.result.get('bootstrap.licensor') if licensor in ['theorchard'] \ and bootstrap.result.get( 'bootstrap.skip_grab_reports_files') == 'True': next_task_requires = bootstrap elif licensor in ['theorchard'] \ and bootstrap.result.get( 'bootstrap.skip_grab_reports_files') != 'True': grab_reports_files = schedule( 'grab_reports_files', self.grab_reports_files, requires=[bootstrap]) if grab_reports_files.result.get( 'grab_reports_files.stop'): return next_task_requires = grab_reports_files else: assert licensor == 'sme' sme_copy_from_athena_to_s3 = schedule( 'sme_copy_from_athena_to_s3', self.sme_copy_from_athena_to_s3, requires=[bootstrap]) if sme_copy_from_athena_to_s3.result.get( 'sme_copy_from_athena_to_s3.stop'): return next_task_requires = sme_copy_from_athena_to_s3 source_files = schedule( 'source_files', self.source_files, requires=[next_task_requires]) load_staging_raw_table = schedule( 'load_staging_raw_table', self.load_staging_raw_table, requires=[source_files]) schedule( 'set_status_to_ingested', self.set_status_to_ingested, requires=[load_staging_raw_table]) return True def contextified_feed_name(self, context): """Get feed_name in context. Args: context (dict): The context of the flow. Returns: str: Contextified feed name. """ assert 'licensor' in context, 'There is no licensor in context' licensor = context['licensor'] assert licensor in config.licensors, \ 'There is no such licensor in config file' return '_'.join([self.feed_name, licensor]) def workflow_id(self, initial_context): """Generate workflow id. Args: initial_context (dict): The initial context for the flow. Returns: str: A unique identifier for a workflow being executed. """ licensor = initial_context['licensor'] date = initial_context['context_date'] return '{flow_name}-{licensor}-{date}'.format( flow_name=self.name, licensor=licensor, date=date) @property def bootstrap(self): """Bootstrap initial configuration.""" return self.create( name='bootstrap', tasks=base.SyncRunner( tasks.bootstrap.fill( namespace='bootstrap', date='context_date', reload='reload', licensor='licensor', skip_grab_reports_files='skip_grab_reports_files'))) @property def source_files(self): """Get list of source files.""" return self.create( name='source_files', tasks=base.SyncRunner( tasks.source_files.fill( namespace='source_files', s3_bucket='bootstrap.s3_bucket', s3_archive_path='bootstrap.s3_archive_path', licensor='bootstrap.licensor'))) @property def sme_copy_from_athena_to_s3(self): """Bootstrap initial configuration.""" return self.create( name='sme_copy_from_athena_to_s3', tasks=base.SyncRunner( youtube_tasks.sme_copy_from_athena_to_s3.fill( namespace='sme_copy_from_athena_to_s3', date='bootstrap.date', report_status_name='bootstrap.feed_name', sme_athena_database=param.StaticParam( config.sme_athena_database), sme_athena_temp_database=param.StaticParam( config.sme_athena_temp_database), sme_athena_source_table=param.StaticParam( config.youtube_report_full_name), destination_s3_bucket='bootstrap.s3_bucket', destination_s3_path='bootstrap.s3_archive_path', athena_workgroup=param.StaticParam( config.athena_workgroup), aws_region=param.StaticParam(config.athena_aws_region), ))) @property def grab_reports_files(self): """Archive report files to archive location.""" return self.create( name='grab_reports_files', schedule_to_start=48000, tasks=base.SyncRunner( youtube_tasks.grab_reports_files.fill( namespace='grab_reports_files', report_name=StaticParam(config.youtube_report_full_name), report_status_name='bootstrap.feed_name', date='bootstrap.date', archive_path='bootstrap.s3_dir_path', credentials_path='bootstrap.credentials_path', api_service_name=StaticParam( config.youtube_reporting_api_service_name), api_version=StaticParam( config.youtube_reporting_api_version), jobs_meta_path=StaticParam(config.jobs_meta_path), cms_dict='bootstrap.cms_dict', gz=param.StaticParam(False)))) load_staging_raw_table = YouTubeAssetSL.load_activity( feed_name=config.feed_name, secrets_path=config.secrets_path, sql_loader=SQLLoader(__file__), requirements=dict( date='bootstrap.date', source_files_dict='source_files.source_files_dict', s3_dir_path='bootstrap.s3_dir_path', feed_name='bootstrap.feed_name', staging_raw_table_name=StaticParam( config.snowflake['staging_raw']), licensor='bootstrap.licensor')) @property def set_status_to_ingested(self): """Set overall feed status to INGESTED.""" return self.create( name='set_status_to_ingested', schedule_to_start=48000, tasks=base.SyncRunner( overall_status_tasks.set_overall_status.fill( namespace='set_status_to_ingested', feed_name='bootstrap.feed_name', date='bootstrap.date', status=param.StaticParam( garcon_feed_status.STATUS_INGESTED))))