"""data_landing_zone workflow.""" from garcon import runner from data_landing_zone.flows.base_flow import BaseFlow from data_landing_zone.flows.spotify import config from data_landing_zone.flows.spotify import tasks class Flow(BaseFlow): """Class representing the workflow.""" timeout = 12 * 60 def __init__(self): """Initialize flow object.""" super(Flow, self).__init__( flow_name=config.SWF_FLOW_NAME, version=config.SWF_FLOW_VERSION) def decider(self, schedule): """Activity decider. Args: schedule (callable): The scheduler method. """ bootstrap = schedule('bootstrap', self.bootstrap) if bootstrap.result.get('bootstrap.stop'): return schedule( 'grab_drop_files', self.grab_drop_files, requires=[bootstrap]) @property def bootstrap(self): """Bootstrap initial configuration.""" return self.create( name='bootstrap', tasks=runner.Sync( tasks.bootstrap.fill( namespace='bootstrap', date='context_date', reload='reload', reports='reports', licensors='licensors'))) @property def grab_drop_files(self): """Request Spotify API and archive files on s3.""" return self.create( name='grab_drop_files', generators=[self.grab_drop_files_generator], schedule_to_start=48000, tasks=runner.Sync( tasks.grab_drop_files.fill( namespace='grab_drop_files', feed_name='feed_name', date='bootstrap.date', report_name='report_name', archive_path='archive_path', licensor='licensor'))) def grab_drop_files_generator(self, context): """Generate parameters for grab_drop_files activity. Args: context (dict): The current context. Yields: dict: Dictionary with report_name and feed name, which will be used for writing statuses in DynamoDB. """ reports_status_names = context['bootstrap.reports_status_names'] archive_paths = context['bootstrap.archive_paths'] for licensor, report_names in reports_status_names.items(): for report_name, feed_name in report_names.items(): yield dict( report_name=report_name, feed_name=feed_name, archive_path=archive_paths[licensor][report_name], licensor=licensor)