import logging import time from datetime import datetime from datetime import timedelta import os import boto3 from ssavva.amazon_unlimited.backfilling import config swf_client = boto3.client( 'swf', aws_access_key_id=os.environ.get('AWS_ACCESS_KEY_ID'), aws_secret_access_key=os.environ.get('AWS_SECRET_ACCESS_KEY')) logger = logging.getLogger(__name__) def can_run_flow(domain, flow_type_name): active_executions = _list_active_runs(domain) return len( [execution for execution in active_executions['executionInfos'] if execution['workflowType']['name'] == flow_type_name] ) < config.parallel_ingestions_number def _list_active_runs(domain): today = datetime.today() start_date, end_date = today - timedelta(days=2), today response = swf_client.list_open_workflow_executions( domain=domain, startTimeFilter={ 'oldestDate': start_date, 'latestDate': end_date}) return response def _get_execution_run_id(domain, flow_type_name, workflow_id): number_of_retries = 10 try_number = 1 while try_number <= number_of_retries: all_active_executions = _list_active_runs(domain) flow_executions = [ execution for execution in all_active_executions['executionInfos'] if execution['workflowType']['name'] == flow_type_name] if flow_executions: for execution in flow_executions: if execution['execution']['workflowId'] == workflow_id: return execution['execution']['runId'] time.sleep(5) raise Exception('Workflow start timeout') def _wait_execution(domain, workflow_id, run_id): while True: response = swf_client.describe_workflow_execution( domain=domain, execution={ 'workflowId': workflow_id, 'runId': run_id } ) if response['executionInfo']['executionStatus'] == 'CLOSED': return response['executionInfo']['closeStatus'] == 'COMPLETED' time.sleep(60) def wait_workflow_execution(domain, flow_type_name, workflow_id): run_id = _get_execution_run_id(domain, flow_type_name, workflow_id) logger.info('Waiting for execution: {} {}'.format(workflow_id, run_id)) return _wait_execution(domain, workflow_id, run_id) if __name__ == '__main__': r = _list_active_runs(config.SWF_AGGREGATION_DOMAIN) print(r)