"""Youtube Video and Claim Summary Workflow.""" import datetime from garcon.param import StaticParam from snowflake_connector.etl_connector import SQLLoader from feed_ingestion.flows import base from feed_ingestion.flows.youtube_video_and_claim_summary import config, tasks from feed_ingestion.flows.youtube_video_and_claim_summary.stage_loader import \ YouTubeVideoAndClaimSummarySL from feed_ingestion.tasks.s3_tasks import copy_files class Flow(base.FlowBase, base.FlowConfigMixin, base.FlowLoadFactMixinSF): """Class representing the workflow.""" def __init__(self): """Initialize flow object.""" super(Flow, self).__init__(config.feed_name, config.feed_version) def decider(self, schedule): """Activity decider. Args: schedule (callable): The scheduler method. """ bootstrap = schedule('bootstrap', self.bootstrap) if bootstrap.result.get('bootstrap.stop') is True: return grab_drop_files = schedule( 'grab_drop_files', self.grab_drop_files, requires=[bootstrap]) if grab_drop_files.result.get('grab_drop_files.stop') is True: return load_staging_raw_table = schedule( 'load_staging_raw_table', self.load_staging_raw_table, requires=[grab_drop_files]) schedule( 'set_status_to_ingested', self.set_status_to_ingested, requires=[load_staging_raw_table]) return True def workflow_id(self, initial_context): """Generate workflow id. Args: initial_context (dict): The initial context for the flow. Returns: str: A unique identifier for a workflow being executed. """ report_name = initial_context['report_name'] flow_name = '_'.join([self.name, report_name]) if 'context_date' not in initial_context: date = datetime.datetime.today().strftime('%Y-%m-%d') else: date = initial_context['context_date'] return '{flow_name}-{date}'.format( flow_name=flow_name, date=date) def contextified_feed_name(self, context): """Get feed_name in context. Args: context (dict): The context of the flow. Returns: str: Contextified feed name. """ assert 'report_name' in context, 'There is no report_name in context' return '_'.join([self.feed_name, context['report_name']]) @property def bootstrap(self): """Bootstrap initial configuration.""" return self.create( name='bootstrap', tasks=base.SyncRunner( tasks.bootstrap.fill( namespace='bootstrap', date='context_date', report_name='report_name', reload='reload'))) @property def grab_drop_files(self): """Archive needed files from the drop location to archive location.""" return self.create( name='grab_drop_files', tasks=base.SyncRunner( copy_files.fill( namespace='grab_drop_files', s3_archive_path='bootstrap.s3_archive_path', s3_download_path='bootstrap.s3_download_path', source_files_dict='bootstrap.source_files_dict', need_all_files=StaticParam(True)))) load_staging_raw_table = YouTubeVideoAndClaimSummarySL.load_activity( feed_name=config.feed_name, secrets_path=config.secrets_path, sql_loader=SQLLoader(__file__), requirements=dict( date='bootstrap.date', report_name='bootstrap.report_name', staging_raw_table_name='bootstrap.staging_raw_table', report_status_name='bootstrap.report_status_name', source_files_dict='bootstrap.source_files_dict', s3_dir_path='bootstrap.s3_archive_path'))