"""Projections ETL tasks tests.""" from datetime import date from unittest.mock import Mock from unittest.mock import patch import pytest from flows.projections import config from flows.projections import queries from flows.projections import status from flows.projections import tasks @pytest.fixture def correlation_id(): """Set correlation_id fixture.""" return '13536586-b8b7-11e6-8c7b-acbc32ca6ded' @patch('flows.projections.tasks.log') @patch('flows.projections.tasks.etl_util') def test_bootstrap(util_mock, log_mock, correlation_id): """Test bootstrap task.""" # prepare test data workflow_run_id = 'run_id' s3_bucket = 'test_bucket' s3_key = 'test/key.csv' upc = '889845077732' file_date = '20161130064705' projection_type = 'regular' drop_url = 's3://{bucket}/{key}'.format(bucket=s3_bucket, key=s3_key) # mock required functions util_mock.parse_file_name.return_value = (upc, file_date) # call test function response = tasks.bootstrap( Mock(), correlation_id, workflow_run_id, s3_bucket, s3_key, projection_type) # check function calls util_mock.parse_file_name.assert_called_once_with(s3_key) log_mock.create.assert_called_once_with( correlation_id, workflow_run_id, upc, file_date, 'projections_revenue_etl_log') assert response['upc'] == upc assert response['drop_url'] == drop_url assert response['revenue_table'] == 'projections_revenue' assert response['transaction_types'] == config.TRANSACTION_TYPE @patch('flows.projections.tasks.log') def test_bootstrap_original_projection(log_mock, correlation_id): """Test bootstrap task for original_projection.""" # prepare test data workflow_run_id = 'run_id' s3_bucket = 'test_bucket' upc = '889845077732' file_date = '20161130064705' s3_key = 'test/{}-{}.csv'.format(upc, file_date) drop_url = 's3://{bucket}/{key}'.format(bucket=s3_bucket, key=s3_key) projection_type = 'original' response = tasks.bootstrap( Mock(), correlation_id, workflow_run_id, s3_bucket, s3_key, projection_type) log_mock.create.assert_called_once_with( correlation_id, workflow_run_id, upc, file_date, 'original_projections_revenue_etl_log') assert response['upc'] == upc assert response['drop_url'] == drop_url assert response['revenue_table'] == 'original_projections_revenue' assert response['transaction_types'] == \ config.ORIGINAL_PROJECTION_TRANSACTION_TYPE @patch('flows.projections.tasks.log') @patch('flows.projections.tasks.s3') def test_archive_dropped_csv(s3, log, correlation_id): """Test archive_dropped_csv task.""" bucket = 'test-bucket' archive_key = 'some/location/archives/test-file' archive_url = 's3://test-bucket/' + archive_key archive_obj = Mock() drop_key = 'some/location/drop/test-file' drop_url = 's3://test-bucket/' + drop_key drop_obj = Mock() drop_obj.bucket_name = bucket drop_obj.key = drop_key s3_objects = {archive_url: archive_obj, drop_url: drop_obj} s3.get_object = lambda x: s3_objects[x] results = tasks.archive_dropped_csv( None, archive_key, bucket, correlation_id, drop_url, 'testlog') assert results['archive_url'] == archive_url assert drop_obj.delete.called archive_obj.copy.assert_called_once_with({ 'Bucket': bucket, 'Key': drop_key}) log.update_status.assert_called_once_with( correlation_id, status.DROP_FILE_ARCHIVED, 'testlog') @patch('flows.projections.tasks.etl_util') @patch('flows.projections.tasks.log') @patch('flows.projections.tasks.util') @pytest.mark.parametrize('projection_type', ['regular', 'original']) def test_determine_anchor_date( util, log, etl_util, correlation_id, projection_type): """Test determine_anchor_date task.""" upc = '54321' etl_util.get_upc_projection_anchor_date.return_value = date(2012, 12, 21) results = tasks.determine_anchor_date( None, correlation_id, upc, 'testlog', projection_type) assert results['anchor_date'] == '2012-12-21' assert 'stop' not in results etl_util.get_upc_projection_anchor_date.assert_called_once_with( upc, projection_type) log.update_status.assert_called_once_with( correlation_id, status.ANCHOR_DATE_DETERMINED, 'testlog') assert not util.send_sns_message.called @patch('flows.projections.tasks.etl_util') @patch('flows.projections.tasks.log') @patch('flows.projections.tasks.util') @pytest.mark.parametrize('projection_type', ['regular', 'original']) def test_determine_anchor_date_none( util, log, etl_util, correlation_id, projection_type): """Test determine_anchor_date task when there is no anchor date.""" upc = '54321' etl_util.get_upc_projection_anchor_date.return_value = None results = tasks.determine_anchor_date( None, correlation_id, upc, 'testlog', projection_type) assert results['anchor_date'] is None assert results['stop'] is True etl_util.get_upc_projection_anchor_date.assert_called_once_with( upc, projection_type) log.update_status.assert_called_once_with( correlation_id, status.ANCHOR_DATE_UNDETERMINED, 'testlog') assert util.send_sns_message.called @patch('flows.projections.tasks.log') @patch('flows.projections.tasks.util') @patch('flows.projections.tasks.queries') @patch('flows.projections.tasks.datastore') def test_create_temp_table( datastore_mock, queries_mock, util_mock, log_mock, correlation_id): """Test create_temp_table task.""" create_sql = 'sql text' queries_mock.CREATE_TEMP_TABLE.format.return_value = create_sql response = tasks.create_temp_table( Mock(), correlation_id, 'testlog', 'testrevenue') util_mock.correlation_id_hex.assert_called_once_with(correlation_id) datastore_mock.execute.assert_called_once_with(create_sql) log_mock.update_status.assert_called_once_with( correlation_id, status.TEMP_TABLE_CREATED, 'testlog') assert 'temp_table_name' in response @patch('flows.projections.tasks.s3') @patch('flows.projections.tasks.etl_util') @patch('flows.projections.tasks.datastore') @patch('flows.projections.tasks.log') @patch('flows.projections.tasks.queries') def test_download_to_db_task( queries, log_mock, datastore_mock, utils_mock, s3_mock, database_context, correlation_id): """Test of download_to_db_task.""" drop_url = 's3://bucket/key' temp_table_name = 'temp_table' upc = '123456789012' query = 'query' dummy_row = [1, 2, 3, 4] params = { 'activity': Mock(), 'correlation_id': correlation_id, 'drop_url': drop_url, 'anchor_date': Mock(), 'upc': upc, 'temp_table_name': temp_table_name, 'log_table': 'testlog', 'transaction_types': {'EST': 12}} datastore_mock.context = database_context s3_mock.download_csv = Mock(return_value=[]) utils_mock.transform = Mock(return_value=[dummy_row]) queries.INSERT_TO_TEMP_TABLE.format = Mock(return_value=query) tasks.download_to_db(**params) database_context._cursor.execute.assert_any_call('START TRANSACTION;') assert database_context._cursor.executemany.any_call_with(query, dummy_row) assert utils_mock.transform.any_call_with( [], params['anchor_date'], params['upc'], params['transaction_types']) log_mock.update_status.assert_called_once_with( correlation_id, status.DATA_DOWNLOADED_TO_TEMP_TABLE, 'testlog') @patch('flows.projections.tasks.datastore') @patch('flows.projections.tasks.log') def test_insert_new_data(log_mock, datastore_mock, correlation_id): """Test of insert_new_data task.""" temp_table_name = 'temp_table' query = queries.COPY_TO_PROD.format( temp_table=temp_table_name, prod_table='testrevenue') tasks.insert_new_data( Mock(), correlation_id, temp_table_name, 'testlog', 'testrevenue') datastore_mock.execute.assert_called_with(query) log_mock.update_status.assert_called_once_with( correlation_id, status.TEMP_TABLE_MADE_LIVE, 'testlog') @patch('flows.projections.tasks.datastore') @patch('flows.projections.tasks.log') def test_drop_temp_table(log_mock, datastore_mock, correlation_id): """Test of drop_temp_table task.""" temp_table_name = 'temp_table' query = queries.DROP_TABLE.format(table_name=temp_table_name) tasks.drop_temp_table(Mock(), correlation_id, temp_table_name, 'testlog') datastore_mock.execute.assert_called_with(query) log_mock.update_status.assert_called_once_with( correlation_id, status.TEMP_TABLE_DROPPED, 'testlog') @patch('flows.projections.tasks.log') def test_set_final_status(log_mock): """Test to set_final_status task.""" tasks.set_final_status(Mock(), correlation_id, 'testlog') log_mock.update_status.assert_called_once_with( correlation_id, status.INGESTED, 'testlog') @patch('flows.projections.tasks.config') @patch('flows.projections.tasks.util') def test_send_success_notification(util_mock, config_mock, correlation_id): """Test send_success_notification task.""" upc = '889845077732' config_mock.SNS_TOPIC_ARN = 'test_arn' config_mock.SNS_SOURCE = 'test_source' config_mock.SNS_ACTION_SUCCESS = 'test_action' tasks.send_success_notification(Mock(), correlation_id, upc) assert util_mock.send_sns_message.called