"""Help functions for Theatrical integration test.""" from datetime import datetime from datetime import timedelta from subprocess import Popen import time from boto.exception import SWFResponseError from boto.s3.connection import Bucket from boto.s3.connection import Key from boto.s3.connection import S3Connection import boto.swf.layer1 as swf from flows import datastore from flows.theatrical import config as flow_config from integration_tests.flows.theatrical import config from integration_tests.flows.theatrical import queries def _prepare_s3(): """Upload csv files to S3.""" for file in config.source_files_data: file_bucket = Bucket(S3Connection(), config.test_bucket_name) key_object = Key(file_bucket) key_object.key = file['s3_path'] with open(file['local_path'], mode='r') as csv_file: key_object.set_contents_from_string(csv_file.read()) def _prepare_mysql(): """Clear required db tables.""" datastore.execute('DROP TABLE IF EXISTS test_theatrical_revenue_etl_log;') datastore.execute( 'CREATE TABLE test_theatrical_revenue_etl_log ' 'LIKE theatrical_revenue_etl_log;') datastore.execute('DROP TABLE IF EXISTS test_theatrical_revenue_raw;') datastore.execute( 'CREATE TABLE test_theatrical_revenue_raw ' 'LIKE theatrical_revenue_raw;') datastore.execute('DROP TABLE IF EXISTS test_theatrical_revenue;') datastore.execute( 'CREATE TABLE test_theatrical_revenue ' 'LIKE theatrical_revenue;') def prepare_data(): """Prepare test data.""" _prepare_s3() _prepare_mysql() def execute_flow(): """Execute Theatrical workflow.""" try: try: swf.Layer1().terminate_workflow_execution( config.domain, flow_config.SWF_WORKFLOW_ID) except SWFResponseError: # if no flow with the same id running pass # start worker process worker_process = Popen( [config.python_path, config.application_path, 'worker', 'theatrical']) # start decider process decider_process = Popen( [config.python_path, config.application_path, 'decider', 'theatrical']) # waiting for activities initialization time.sleep(20) # start execution process exec_process = Popen( [config.python_path, config.application_path, 'execute', 'theatrical', '-s', config.backfill_start_date, '-e', config.backfill_end_date]) exec_process.wait() # check that log record was created log_check = queries.assert_queries['logs_count'] logs = datastore.execute(log_check['sql']) assert logs.rowcount == log_check['check_value'], log_check[ 'error_message'] # get workflow run id run_id = [log[0] for log in logs][0] # wait for workflow execution end max_time = datetime.now() + timedelta( seconds=config.flow_execution_timeout) while datetime.now() < max_time: execution = swf.Layer1().describe_workflow_execution( domain=config.domain, workflow_id=flow_config.SWF_WORKFLOW_ID, run_id=run_id) time.sleep(1) if execution['executionInfo'].get('executionStatus') != 'CLOSED': time.sleep(10) if datetime.now() > max_time: raise TimeoutError('Theatrical flow execution timeout') else: break except Exception as e: print(e) raise finally: worker_process.kill() decider_process.kill()