"""Workflow runner for Python based integration tests.""" from datetime import datetime from datetime import timedelta import json import os import subprocess import time import boto3 import integration_tests.python_integration_tests.youtube_weekly.config \ as config from feed_ingestion.conf.config import BOTO3_CONFIG TIMEOUT = 15 # 15 minutes to run def run_workflow(workflow_type, flow_category): """Run workflow. Args: workflow_type (str): type of the flow, needed to get info from config. flow_category (str): category of the flow (one of the __all__ = [ 'apple_music', 'apple_music_streams', 'apple_music_demographics_only', 'itunes', 'qq', 'deezer', 'youtube_monthly', 'youtube_weekly', 'proper', 'proper_incoming'] list). Returns: str: closeStatus (e.g. COMPLETED) of the SWF workflow. """ swf_client = boto3.client('swf', config=BOTO3_CONFIG) try: executions = swf_client.list_open_workflow_executions( domain=config.swf_domain, startTimeFilter={ 'oldestDate': datetime.now() - timedelta(hours=24) }, executionFilter={ 'workflowId': config.workflow_ids[workflow_type] } )['executionInfos'] for execution in executions: swf_client.terminate_workflow_execution( domain=config.swf_domain, workflowId=execution['execution']['workflowId'], runId=execution['execution']['runId'], ) print('Previous run of this test workflow terminated') print('spawn a Decider') decider_proc = subprocess.Popen(['garcon', 'decider', flow_category]) # in order to register if it's the first run in the 'test' domain time.sleep(30) print('spawn a Worker') worker_proc = subprocess.Popen(['garcon', 'worker', flow_category]) print('Executor kick off the flow') executor_proc = subprocess.Popen( ['garcon', 'exec', flow_category, '-o', 'out.txt', '-c', config.exec_contexts[workflow_type]]) executor_proc.wait() # read in Executor details flow_details = json.load(open('out.txt')) print('Polling until Workflow is complete') max_time = datetime.now() + timedelta(minutes=TIMEOUT) while datetime.now() < max_time: if datetime.now() >= max_time: raise Exception('Flow execution takes more than 15 min') execution = swf_client.describe_workflow_execution( domain=config.swf_domain, workflow_id=flow_details['workflow_id'], run_id=flow_details['run_id']) time.sleep(1) if execution['executionInfo'].get('executionStatus') != 'CLOSED': time.sleep(10) else: break return execution['executionInfo'].get('closeStatus') except Exception as e: print(e) raise finally: # shut down print('Killing decider and worker') decider_proc.kill() worker_proc.kill() print('Removing out.txt') os.remove('out.txt') print('Workflow closed')