"""SWF related helpers.""" import datetime from urllib.parse import quote_plus import boto3 from feed_ingestion import getconf from feed_ingestion.conf.config import BOTO3_CONFIG SWF_FINAL_STATUSES_SUCCESS = frozenset([ 'COMPLETED', 'CONTINUED_AS_NEW', ]) SWF_FINAL_STATUSES_FAILURE = frozenset([ 'FAILED', 'CANCELED', 'TERMINATED', 'TIMED_OUT', ]) SWF_FINAL_STATUSES = SWF_FINAL_STATUSES_SUCCESS | SWF_FINAL_STATUSES_FAILURE def count_running_workflows_by_type( domain, type_name, num_of_days_back=7): """Count the number of running workflows with a certain workflow type. Note that the num_of_days_back param should be always greater than the workflow execution timeout. Args: domain (str): Domain of the workflow. type_name (str): Name of the workflow type. num_of_days_back (int): Number of days after which this check should perform. Returns: int: Number of open executions. """ client = boto3.client('swf', config=BOTO3_CONFIG) oldest_date = datetime.date.today() - datetime.timedelta( days=num_of_days_back) response = client.count_open_workflow_executions( domain=domain, startTimeFilter={ 'oldestDate': datetime.datetime.combine( oldest_date, datetime.datetime.min.time()) }, typeFilter={ 'name': type_name }) return response.get('count') def has_nonconcurrent_workflows_running(flow): """Check if there is nonconcurrent workflows running. Note that this check will be skipped if current workflow does not exist in the list of nonconcurrent workflow types. Also, num_of_days_back is not passed in, because no nonconcurrent workflows has execution start to close timeout greater than 7 days. Args: flow (module): Garcon flow module. Returns: bool: True if other nonconcurrent workflow(s) is running. """ has_nonconcurrent_workflows = False nonconcurrent_workflow_types = getconf( 'nonconcurrent_workflow_types')['workflow_types'] if flow.name in nonconcurrent_workflow_types: for workflow_type in nonconcurrent_workflow_types: if count_running_workflows_by_type( flow.domain, workflow_type): has_nonconcurrent_workflows = True break return has_nonconcurrent_workflows def get_names_of_running_workflows_by_type( domain, type_name, num_of_days_back=7): """Get list of flow names by domain and flow_type. Note that the num_of_days_back param should be always greater than the workflow execution timeout. Args: domain (str): Domain of the workflow. type_name (str): Name of the workflow type. num_of_days_back (int): Number of days after which this check should perform. Returns: list: List of open executions. """ client = boto3.client('swf', config=BOTO3_CONFIG) oldest_date = datetime.date.today() - datetime.timedelta( days=num_of_days_back) response = client.list_open_workflow_executions( domain=domain, startTimeFilter={ 'oldestDate': datetime.datetime.combine( oldest_date, datetime.datetime.min.time()) }, typeFilter={ 'name': type_name }) # get only full flow name that we see in swf result = [ flow['execution']['workflowId'] for flow in response['executionInfos']] return result def list_closed_swf_executions( from_: datetime.datetime, to: datetime.datetime, swf_domain: str, execution_type: str, filter_by_statuses=SWF_FINAL_STATUSES_SUCCESS ): """Load executions completed in the given timeframe.""" swf_client = boto3.client('swf', config=BOTO3_CONFIG) executions = swf_client.list_closed_workflow_executions( domain=swf_domain, startTimeFilter={ 'latestDate': to, 'oldestDate': from_, }, typeFilter={ 'name': execution_type, }, maximumPageSize=1000, )['executionInfos'] filtered = [ e for e in executions if e['closeStatus'] in filter_by_statuses ] return filtered def tasks_from_events(events): """Build task activities from execution events.""" event_by_id = {e['eventId']: e for e in events} tasks_closed = \ [e for e in events if e['eventType'] == 'ActivityTaskCompleted'] tasks = [] for task_closed in tasks_closed: task_open = event_by_id[ task_closed['activityTaskCompletedEventAttributes'] ['startedEventId'] ] event_scheduled_id = task_open['activityTaskStartedEventAttributes'][ 'scheduledEventId'] task_scheduled = event_by_id[event_scheduled_id] sched_attr = task_scheduled['activityTaskScheduledEventAttributes'] task_name = sched_attr['activityType']['name'] duration = task_closed['eventTimestamp'] - task_open['eventTimestamp'] task = { 'name': task_name, 'duration': duration, 'scheduled': task_scheduled, 'open': task_open, 'closed': task_closed, } tasks.append(task) return tasks def load_all_execution_events(swf_domain, workflow_id, run_id): """Suck all executions events for SWF execution.""" swf_client = boto3.client('swf', config=BOTO3_CONFIG) events = [] next_page_token = None while True: kwargs = dict( domain=swf_domain, execution={ 'workflowId': workflow_id, 'runId': run_id, }, maximumPageSize=1000, ) if next_page_token: kwargs['nextPageToken'] = next_page_token response = swf_client.get_workflow_execution_history(**kwargs) events.extend(response['events']) next_page_token = response.get('nextPageToken') if not next_page_token: break return events def generate_execution_console_url( swf_domain, workflow_id, run_id, aws_region='us-east-1'): """Return URL to the given SWF execution in AWS web-console.""" if not run_id: return None return (f'https://{aws_region}.console.aws.amazon.com' f'/swf/v2/home?region={aws_region}#/domains/{swf_domain}/' f'executions/{quote_plus(workflow_id)}' f'/{quote_plus(run_id)}')