"""iTunes Tickets Workflow.""" from garcon.param import StaticParam from garcon_contrib.dynamo_feed_status import garcon_feed_status from feed_ingestion.flows import base from feed_ingestion.flows.base import FlowBase from feed_ingestion.flows.base import FlowConfigMixin from feed_ingestion.flows.base import FlowLoadRawMixinSF from feed_ingestion.flows.itunes_tickets import config from feed_ingestion.flows.itunes_tickets import tasks from feed_ingestion.tasks import load_raw_table_tasks_sf from feed_ingestion.tasks import overall_status_tasks class Flow(FlowBase, FlowConfigMixin, FlowLoadRawMixinSF): """iTunes Tickets Flow class. This flow downloads iTunes Tickets data and loads to a itunes tickets staging tables. """ def __init__(self): """Initialize a iTunes Tickets workflow.""" super(Flow, self).__init__(config.feed_name, config.feed_version) self.timeout = 20 * 60 # Timeout after 20 minutes def decider(self, schedule): """Orchestrate a iTunes Tickets workflow. Args: schedule (callable): The scheduler method. """ bootstrap = schedule('bootstrap', self.bootstrap) if bootstrap.result.get('bootstrap.stop') is True: return grab_drop_files = schedule( 'grab_drop_files', self.grab_drop_files, requires=[bootstrap]) if grab_drop_files.result.get('grab_drop_files.stop') is True: return # Status updated to DOWNLOADED if files are present set_status_to_downloaded = schedule( 'set_status_to_downloaded', self.set_status_to_downloaded, requires=[grab_drop_files]) process_drop_files = schedule( 'process_drop_files', self.process_drop_files, requires=[set_status_to_downloaded]) create_temp_staging_raw_table = schedule( 'create_temp_staging_raw_table', self.create_temp_staging_raw_table, requires=[process_drop_files]) load_temp_staging_raw_table = schedule( 'load_temp_staging_raw_table', self.load_temp_staging_raw_table, requires=[create_temp_staging_raw_table]) load_staging_raw_table = schedule( 'load_staging_raw_table', self.load_staging_raw_table, requires=[load_temp_staging_raw_table]) set_status_to_populated_raw_table = schedule( 'set_status_to_populated_raw_table', self.set_status_to_populated_raw_table, requires=[load_staging_raw_table]) drop_temp_staging_table = schedule( 'drop_temp_staging_table', self.drop_temp_staging_table, requires=[set_status_to_populated_raw_table]) schedule( 'set_status_to_ingested', self.set_status_to_ingested, requires=[drop_temp_staging_table]) @property def bootstrap(self): """Bootstrap initial configuration.""" return self.create( name='bootstrap', tasks=base.SyncRunner( tasks.bootstrap.fill( namespace='bootstrap', date='context_date', reload='reload'))) @property def grab_drop_files(self): """Grab report and upload it to archive location.""" return self.create( name='grab_drop_files', tasks=base.SyncRunner( tasks.grab_drop_files.fill( namespace='grab_drop_files', date='bootstrap.date', feed_name='bootstrap.feed_name', archive_path='bootstrap.archive_path'))) @property def process_drop_files(self): """Process drop files.""" return self.create( name='process_drop_files', tasks=base.SyncRunner( tasks.process_drop_files.fill( namespace='process_drop_files', date='bootstrap.date', feed_name='bootstrap.feed_name', archive_path='bootstrap.archive_path', preprocessed_path='bootstrap.preprocessed_path', source_files_dict='grab_drop_files.source_files_dict'))) @property def create_temp_staging_raw_table(self): """Create temp staging raw table.""" return self.create( name='create_temp_staging_raw_table', generators=[self.temp_staging_tables_generator], tasks=base.SyncRunner( load_raw_table_tasks_sf.create_temp_staging_raw_table.fill( namespace='create_temp_staging_raw_table', date='bootstrap.date', sfdb_params='sfdb_params', feed_name='bootstrap.feed_name', secrets_path='bootstrap.secrets_path', temp_staging_raw_table='temp_staging_raw_table'))) @property def load_temp_staging_raw_table(self): """Load temp staging raw table.""" return self.create( name='load_temp_staging_raw_table', generators=[self.temp_staging_tables_generator], tasks=base.SyncRunner( load_raw_table_tasks_sf.load_temp_staging_raw_table.fill( namespace='load_temp_staging_raw_table', date='bootstrap.date', sfdb_params='sfdb_params', feed_name='bootstrap.feed_name', kwargs='kwargs', key_dir='key_dir', secrets_path='bootstrap.secrets_path', temp_staging_raw_table='temp_staging_raw_table'))) @property def drop_temp_staging_table(self): """Drop the temporary staging tables.""" return self.create( name='drop_temp_staging_table', generators=[self.temp_staging_tables_generator], tasks=base.SyncRunner( tasks.drop_temp_table.fill( namespace='drop_temp_staging_table', temp_table_name='temp_staging_raw_table', secrets_path='bootstrap.secrets_path'))) def temp_staging_tables_generator(self, context): """Generate parameters for temporary staging tables. Used by the create_temp_staging_raw_table, load_temp_staging_raw_table activity and drop_temp_staging_table. Args: context (dict): The current context. Yields: dict: Dictionary of temporary staging table names. """ preprocessed_path = context['bootstrap.preprocessed_path'] for table_type in config.table_types: key_dir = 's3://{bucket}/{preprocessed_path}{file_name}'.format( bucket=config.data_bucket, preprocessed_path=preprocessed_path, file_name='preprocessed_{}.xml.gz').format(table_type) yield dict( table_type=table_type, temp_staging_raw_table=context[ 'bootstrap.temp_table_names'][table_type], key_dir=key_dir) @property def set_status_to_downloaded(self): """Set overall feed status to DOWNLOADED.""" return self.create( name='set_status_to_downloaded', tasks=base.SyncRunner( overall_status_tasks.set_overall_status.fill( namespace='set_status_to_downloaded', feed_name='bootstrap.feed_name', date='bootstrap.date', status=StaticParam( garcon_feed_status.STATUS_DOWNLOADED)))) @property def set_status_to_populated_raw_table(self): """Set overall feed status to POPULATED_RAW_TABLE.""" return self.create( name='set_status_to_populated_raw_table', tasks=base.SyncRunner( overall_status_tasks.set_overall_status.fill( namespace='set_status_to_populated_raw_table', feed_name='bootstrap.feed_name', date='bootstrap.date', status=StaticParam( garcon_feed_status.STATUS_POPULATED_RAW_TABLE))))