"""elasticsearch_export workflow.""" from garcon import runner from yt_conflict_elasticsearch.flows.base_flow import BaseFlow from yt_conflict_elasticsearch.flows.elasticsearch_export import config from yt_conflict_elasticsearch.flows.elasticsearch_export import generators from yt_conflict_elasticsearch.flows.elasticsearch_export import tasks class Flow(BaseFlow): """Class representing the workflow.""" timeout = 60 * 60 * 12 def __init__(self): """Initialize flow object.""" super(Flow, self).__init__( flow_name=config.SWF_FLOW_NAME, version=config.SWF_FLOW_VERSION) def decider(self, schedule, context): """Activity decider. Args: schedule (callable): The scheduler method. """ bootstrap = schedule('bootstrap', self.bootstrap) if not bootstrap.result.get('bootstrap.date'): return store_conflicts_in_database_as_csv = schedule( 'store_conflicts_in_database_as_csv', self.store_conflicts_in_database_as_csv, requires=[bootstrap]) transform_csv_to_json_conflicts = schedule( 'transform_csv_to_json_conflicts', self.transform_csv_to_json_conflicts, requires=[store_conflicts_in_database_as_csv]) create_elasticsearch_index = schedule( 'create_elasticsearch_index', self.create_elasticsearch_index, requires=[transform_csv_to_json_conflicts]) populate_elasticsearch_index = schedule( 'populate_elasticsearch_index', self.populate_elasticsearch_index, requires=[create_elasticsearch_index]) mark_indexed_conflicts = schedule( 'mark_indexed_conflicts', self.mark_indexed_conflicts, requires=[populate_elasticsearch_index]) remove_responded_conflicts_from_es = schedule( 'remove_responded_conflicts_from_es', self.remove_responded_conflicts_from_es, requires=[mark_indexed_conflicts] ) remove_resolved_conflicts_from_es = schedule( 'remove_resolved_conflicts_from_es', self.remove_resolved_conflicts_from_es, requires=[remove_responded_conflicts_from_es] ) schedule( 'clean_up_s3', self.clean_up_s3, requires=[remove_resolved_conflicts_from_es]) @property def bootstrap(self): """Bootstraps the flow.""" return self.create( name='bootstrap_example', tasks=runner.Sync( tasks.bootstrap.fill( namespace='bootstrap', date='context_date', reload='reload'))) @property def store_conflicts_in_database_as_csv(self): """Store new conflicts as a CSV file in s3.""" return self.create( name='store_conflicts_in_database_as_csv', generators=[generators.conflict_status], tasks=runner.Sync( tasks.task_store_conflicts_in_database_as_csv.fill( namespace='store_conflicts_in_database_as_csv', date='bootstrap.date', conflict_status='conflict_status'))) @property def transform_csv_to_json_conflicts(self): """Transform CSV file into a JSON file of conflicts.""" return self.create( name='transform_csv_to_json_conflicts', generators=[generators.conflict_status], tasks=runner.Sync( tasks.task_transform_csv_to_json_conflicts.fill( namespace='transform_csv_to_json_conflicts', date='bootstrap.date', conflict_status='conflict_status'))) @property def create_elasticsearch_index(self): """Create and populate the Elasticsearch index.""" return self.create( name='create_elasticsearch_index', tasks=runner.Sync( tasks.task_create_elasticsearch_index.fill( namespace='create_elasticsearch_index', date='bootstrap.date'))) @property def populate_elasticsearch_index(self): """Create and populate the Elasticsearch index.""" return self.create( name='populate_elasticsearch_index', generators=[generators.conflict_status], tasks=runner.Sync( tasks.task_populate_elasticsearch_index.fill( namespace='populate_elasticsearch_index', date='bootstrap.date', conflict_status='conflict_status'))) @property def mark_indexed_conflicts(self): """Mark indexed conflicts.""" return self.create( name='mark_indexed_conflicts', tasks=runner.Sync( tasks.task_mark_indexed_conflicts.fill( namespace='mark_indexed_conflicts', date='bootstrap.date'))) @property def clean_up_s3(self): """Delete conflicts files from S3 Bucket.""" return self.create( name='clean_up_s3', tasks=runner.Sync( tasks.clean_up_s3.fill( namespace='clean_up_s3', date='bootstrap.date'))) @property def remove_responded_conflicts_from_es(self): """Remove responded conflicts from ES index.""" return self.create( name='remove_responded_conflicts_from_es', tasks=runner.Sync( tasks.remove_responded_conflicts_from_es.fill( namespace='mark_indexed_conflicts', date='bootstrap.date'))) @property def remove_resolved_conflicts_from_es(self): """Remove resolved conflicts from ES index.""" return self.create( name='remove_resolved_conflicts_from_es', tasks=runner.Sync( tasks.remove_resolved_conflicts_from_es.fill( namespace='remove_resolved_conflicts_from_es', date='bootstrap.date')))