"""Unit tests for util.""" import datetime import decimal import json from unittest.mock import MagicMock from unittest.mock import Mock from unittest.mock import patch import pytest from pytest import raises from flows.cable_calculation import util @pytest.fixture def upcs(): """Flat list of UPCs.""" return ['123', '234', '345', '456', '567'] @pytest.fixture def upcs_batches(): """Nested list of batch UPCs.""" return [['123', '234', '345'], ['456', '567']] @patch('flows.cable_calculation.util.boto3') @patch('flows.cable_calculation.util.flow_config') def test_send_success_notification(config, boto): """Test sending success notification.""" config.SNS_ACTION_SUCCESS = 'test passed' config.SNS_SOURCE = 'py.test' config.SNS_TOPIC_ARN = 'a unit test' sns = Mock() sns.publish.return_value = {'foo': 'bar'} boto.client.return_value = sns correlation_id = '0123-4567-8910-1112' date_end = 'yesterday' date_start = 'last year' upcs = ['123', '456'] result = util.send_success_notification( correlation_id, date_end, date_start, upcs) assert result == {'foo': 'bar'} expected_payload = { 'action': config.SNS_ACTION_SUCCESS, 'correlation_id': correlation_id, 'date_end': date_end, 'date_start': date_start, 'source': config.SNS_SOURCE, 'upcs': upcs} params = sns.publish.call_args_list[0][1] assert params['TopicArn'] == config.SNS_TOPIC_ARN assert params['Subject'] == config.SNS_ACTION_SUCCESS assert json.loads(params['Message']) == expected_payload @patch('flows.cable_calculation.util.boto3') @patch('flows.cable_calculation.util.flow_config') def test_queue_build_cache(config, boto): """Test sending message to SQS to build the cache.""" config.SQS_DROP_CACHE_ACTION = 'test-function' config.SQS_BUILD_CACHE_REGION_NAME = 'unit-test-1' config.SQS_BUILD_CACHE_SOURCE = 'unit-test:digital' config.SQS_BUILD_CACHE_URL = 'https://unittest.example.com/ft-etl' sqs = Mock() sqs.send_message.return_value = {'foo': 'bar'} boto.client.return_value = sqs correlation_id = '0123-4567-8910-1112' date_end = 'yesterday' date_start = 'last year' upcs = ['123', '456'] result = util.queue_build_cache(correlation_id, date_end, date_start, upcs) assert result == {'foo': 'bar'} expected_payload = { 'action': config.SQS_DROP_CACHE_ACTION, 'correlation_id': correlation_id, 'date_end': date_end, 'date_start': date_start, 'source': config.SQS_BUILD_CACHE_SOURCE, 'upcs': upcs} params = sqs.send_message.call_args_list[0][1] assert params['QueueUrl'] == config.SQS_BUILD_CACHE_URL assert json.loads(params['MessageBody']) == expected_payload def test_get_est_dates_for_upc(): """Test get_est_dates_for_upc.""" all_data = [ {'upc': 190374851541, 'est_date': '2016-08-16 00:00:00'}, {'upc': 191018032753, 'est_date': '2016-04-18 16:47:04'}] result = util.get_est_dates_for_upc(all_data, 191018032753) assert result == datetime.datetime.strptime( '2016-04-18 16:47:04', '%Y-%m-%d %H:%M:%S') result = util.get_est_dates_for_upc(all_data, 1234) assert result is None result = util.get_est_dates_for_upc([], 191018032753) assert result is None def test_get_dbo_for_upc(): """Test getting dbo for a upc.""" all_data = [ {'upc': 190374851541, 'date': '2016-11-07', 'dbo': 123.0}, {'upc': 191018032753, 'date': '2016-11-10', 'dbo': 510.0}] result = util.get_dbo_for_upc( all_data, datetime.datetime.strptime( '2016-11-10', '%Y-%m-%d'), 191018032753) assert result == 510.0 result = util.get_dbo_for_upc( all_data, datetime.datetime.strptime( '2016-11-10', '%Y-%m-%d'), 190374851541) assert result is None result = util.get_dbo_for_upc( all_data, datetime.datetime.strptime('2016-11-10', '%Y-%m-%d'), 1234) assert result is None result = util.get_dbo_for_upc( [], datetime.datetime.strptime('2016-11-10', '%Y-%m-%d'), 1234) assert result is None def test_unload_from_raw_table(monkeypatch, upcs, upcs_batches): """Test unload_from_raw_table.""" date_start = '2016-08-02' date_end = '2016-09-25' # Mocking select = Mock() select_format_1 = Mock() select_format_2 = Mock() select.format.side_effect = [select_format_1, select_format_2] cursor_mock = MagicMock() context = MagicMock() context.__enter__ = Mock(return_value=(cursor_mock, MagicMock())) monkeypatch.setattr( util.datastore, 'context', MagicMock(return_value=context)) util.unload_from_raw_table.batch_size = 3 results_raw = util.unload_from_raw_table( upcs, date_start, date_end, select) results = list(results_raw) # Assertion format_calls = select.format.call_args_list for batch, upcs_batch in enumerate(upcs_batches): for upc in upcs_batch: assert upc in format_calls[batch][1]['upc_in_clause'] assert len(results) == len(upcs_batches) cursor_mock.execute.assert_any_call( select_format_1, {'date_start': date_start, 'date_end': date_end}) cursor_mock.execute.assert_any_call( select_format_2, {'date_start': date_start, 'date_end': date_end}) @patch('flows.cable_calculation.util.check_upcs') @patch('flows.cable_calculation.util.datawarehouse') @patch('flows.cable_calculation.util.etl_logger') def test_load_est_date_for_releases( log, datawarehouse, check_upcs, upcs, upcs_batches): """Test load_dbo_from_theatrical_revenue.""" datawarehouse.execute.side_effect = ( [('123456789', '2016-08-08')], [('888845555', '2016-09-17')]) select_q = Mock() select_q.format.return_value = 'SELECT query a, b' util.load_est_date_for_releases.batch_size = 3 response_raw = util.load_est_date_for_releases(upcs=upcs, select=select_q) response = list(response_raw) # Assertions assert len(response) == len(upcs_batches) for batch in upcs_batches: check_upcs.assert_any_call(batch) select_q.format.assert_any_call(upcs=', '.join(batch)) datawarehouse.execute.assert_called_with('SELECT query a, b') assert datawarehouse.execute.call_count == len(upcs_batches) assert response == [ {'123456789': '2016-08-08'}, {'888845555': '2016-09-17'}] @patch('flows.cable_calculation.util.check_upcs') @patch('flows.cable_calculation.util.process_dbo') @patch('flows.cable_calculation.util.datastore') @patch('flows.cable_calculation.util.etl_logger') def test_load_dbo_from_theatrical_revenue( log, datastore, process_dbo, check_upcs, database_context, upcs, upcs_batches): """Test load_dbo_from_theatrical_revenue.""" select = 'SELECT query' select_q = Mock() select_q.format.return_value = select expected_rows = ('row1', 'row2') expected_formatted = [dict(foo='formatted1'), dict(bar='formatted2')] # Stubbing datastore.context = database_context database_context._cursor.fetchall.return_value = expected_rows process_dbo.side_effect = expected_formatted util.load_dbo_from_theatrical_revenue.batch_size = 3 response_raw = util.load_dbo_from_theatrical_revenue( upcs=upcs, select=select_q) response = list(response_raw) # Assertions for batch in upcs_batches: check_upcs.assert_any_call(batch) select_q.format.assert_any_call(upcs=', '.join(batch)) database_context._cursor.execute.assert_called_with(select) process_dbo.assert_called_with(expected_rows) assert response == expected_formatted @pytest.mark.parametrize('rows, expected', [ (( (190374851541, datetime.date(2016, 11, 7), decimal.Decimal('123.00')), (191018032753, datetime.date(2016, 11, 10), decimal.Decimal('510.00')), (190374851541, datetime.date(2016, 11, 15), decimal.Decimal('212.00')), ), { '190374851541': { '2016-11-07': 123.0, '2016-11-15': 212.0 }, '191018032753': { '2016-11-10': 510.0 }, }), ((), {}) # Empty RS ]) def test_format_dbo(rows, expected): """Test format_dbo.""" res = util.format_dbo(rows) assert res == expected @pytest.mark.parametrize('rows, expected', [ (( (datetime.date(2016, 11, 8), 'foo'), (datetime.date(2016, 11, 9), 'bar'), (datetime.date(2016, 11, 10), 'blah'), (datetime.date(2016, 11, 15), 'hey') ), (datetime.date(2016, 11, 8), datetime.date(2016, 11, 15))), (( (datetime.date(2016, 9, 23), 'nevermind'), ), (datetime.date(2016, 9, 23), datetime.date(2016, 9, 23))), ((), (None, None)) # Empty RS ]) def test_get_date_range(rows, expected): """Test get_date_range.""" res = util.get_date_range(rows) assert res == expected @pytest.mark.parametrize('rows, expected', [ (( (190374851541, datetime.date(2016, 11, 8), decimal.Decimal('214.84')), (190374851541, datetime.date(2016, 11, 9), decimal.Decimal('456.65')), (190374851541, datetime.date(2016, 11, 10), decimal.Decimal('4564.1')), (190374851541, datetime.date(2016, 11, 15), decimal.Decimal('899.0')), (191018032753, datetime.date(2016, 11, 9), decimal.Decimal('23.0')), (191018032753, datetime.date(2016, 11, 10), decimal.Decimal('459.0')), (191018032753, datetime.date(2016, 11, 11), decimal.Decimal('7895.0')) ), { '190374851541': [ (datetime.date(2016, 11, 8), decimal.Decimal('214.84')), (datetime.date(2016, 11, 9), decimal.Decimal('456.65')), (datetime.date(2016, 11, 10), decimal.Decimal('4564.1')), (datetime.date(2016, 11, 15), decimal.Decimal('899.0')) ], '191018032753': [ (datetime.date(2016, 11, 9), decimal.Decimal('23.0')), (datetime.date(2016, 11, 10), decimal.Decimal('459.0')), (datetime.date(2016, 11, 11), decimal.Decimal('7895.0')) ] }), (( (190374851541, datetime.date(2016, 11, 8), decimal.Decimal('214.84')), ), { '190374851541': [ (datetime.date(2016, 11, 8), decimal.Decimal('214.84'))]}), ((), {}) # Empty RS ]) def test_dbo_rows_to_buckets(rows, expected): """Test dbo_rows_to_buckets.""" res = util.dbo_rows_to_buckets(rows) assert res == expected @pytest.mark.parametrize('rows, expected', [ ([ (datetime.date(2016, 11, 8), decimal.Decimal('214.84')), (datetime.date(2016, 11, 9), decimal.Decimal('456.65')), (datetime.date(2016, 11, 10), decimal.Decimal('4564.1')), (datetime.date(2016, 11, 15), decimal.Decimal('899.0')) ], { '2016-11-08': 214.84, '2016-11-09': 456.65, '2016-11-10': 4564.1, '2016-11-15': 899.0 }), ([], {}) # Empty RS ]) def test_dbo_rows_to_dict(rows, expected): """Test dbo_rows_to_dict.""" res = util.dbo_rows_to_dict(rows) assert res == expected @pytest.mark.parametrize('rows, start, end, expected', [ ({ '2016-11-09': 456.65, '2016-11-10': 4564.1, '2016-11-15': 899.0 }, datetime.date(2016, 11, 9), datetime.date(2016, 11, 15), { '2016-11-09': 456.65, '2016-11-10': 5020.75, '2016-11-11': 5020.75, '2016-11-12': 5020.75, '2016-11-13': 5020.75, '2016-11-14': 5020.75, '2016-11-15': 5919.75, }), ({ '2016-11-08': 214.84 }, datetime.date(2016, 11, 8), datetime.date(2016, 11, 8), { '2016-11-08': 214.84 }) ]) def test_fill_dbo_gaps(rows, start, end, expected): """Test fill_dbo_gaps.""" res = util.fill_dbo_gaps(rows, start, end) assert res == expected @pytest.mark.parametrize('rows, expected', [ (( (190374851541, datetime.date(2016, 11, 9), decimal.Decimal('456.65')), (190374851541, datetime.date(2016, 11, 10), decimal.Decimal('4564.1')), (190374851541, datetime.date(2016, 11, 15), decimal.Decimal('899.0')), (191018032753, datetime.date(2016, 9, 21), decimal.Decimal('23.0')) ), { '190374851541': { '2016-11-09': 456.65, '2016-11-10': 5020.75, '2016-11-11': 5020.75, '2016-11-12': 5020.75, '2016-11-13': 5020.75, '2016-11-14': 5020.75, '2016-11-15': 5919.75, }, '191018032753': { '2016-09-21': 23.0 }}) ]) def test_process_dbo(rows, expected): """Test process_dbo.""" res = util.process_dbo(rows) assert res == expected @pytest.mark.parametrize('rule_found, expected_split_amount', [ (True, 0.45), (False, 0.00) ]) def test_calculate_single_row(rule_found, expected_split_amount, monkeypatch): """Test calculate_single_row.""" dbo = { '888845612': { '2016-11-07': 123.0, '2016-11-10': 510.0, '2016-09-16': 899.0 }, '888845613': { '2016-09-09': 23.0, '2016-09-10': 459.0, '2016-09-11': 7895.0 } } est_dates = {'888845612': '2016-08-22'} expected_store = Mock(id=123) row = ( 12, 'HD', 'FOD', None, None, datetime.date(2016, 9, 15), 3.95, 888845612, 0, datetime.date(2016, 9, 16), 1, 42, 0, 2, 1, 0 ) operator_id, resolution, content_type, theatrical_release_date, \ home_video_release_date, vod_start_of_window, revenue, \ upc, dbo_ignored, date, transactions, transaction_type_id, \ paid, format_id, country_id, orchard_amount = row # Mocking if rule_found: expected_split_id = 'foo' split_rule_mock = Mock() split_rule_mock.apply.return_value = expected_split_amount split_rule_mock.id = expected_split_id else: split_rule_mock = None expected_split_id = '' get_store_by_provider_mock = MagicMock(return_value=expected_store) monkeypatch.setattr( util.stores, 'get_store_by_provider', get_store_by_provider_mock) get_split_rule_mock = MagicMock(return_value=split_rule_mock) monkeypatch.setattr( util.stores, 'get_split_rule', get_split_rule_mock) expected_row = ( resolution, content_type, theatrical_release_date, home_video_release_date, vod_start_of_window, revenue, upc, dbo_ignored, date, transactions, transaction_type_id, paid, format_id, country_id, expected_split_amount, expected_split_id) apply_params = dict( gross=revenue, dbo=dbo.get(str(upc), {}).get(date.isoformat(), 0), vod_date=vod_start_of_window, est_date=datetime.datetime.strptime( est_dates.get(str(upc)), '%Y-%m-%d').date(), unit_price=float(revenue) / int(transactions) ) res = util.calculate_single_row(est_dates, dbo, row) # Assertion get_store_by_provider_mock.assert_called_with(operator_id) get_split_rule_mock.assert_called_with(date, expected_store.id) if rule_found: split_rule_mock.apply.assert_called_with(**apply_params) assert res == expected_row def test_calculate_single_row_no_store(monkeypatch): """Test calculate_single_row.""" dbo = { '888845612': { '2016-11-07': 123.0, '2016-11-10': 510.0, '2016-09-16': 899.0 }, '888845613': { '2016-09-09': 23.0, '2016-09-10': 459.0, '2016-09-11': 7895.0 } } est_dates = {'888845612': '2016-08-22'} expected_store = None row = ( 12, 'HD', 'FOD', None, None, '2016-09-15', 3.95, 888845612, 0, '2016-09-16', 1, 42, 0, 2, 1, 0 ) operator_id, resolution, content_type, theatrical_release_date, \ home_video_release_date, vod_start_of_window, revenue, \ upc, dbo_ignored, date, transactions, transaction_type_id, \ paid, format_id, country_id, orchard_amount = row # Mocking expected_split_amount = 0 expected_split_id = 'Not applied' get_store_by_provider_mock = MagicMock(return_value=expected_store) monkeypatch.setattr( util.stores, 'get_store_by_provider', get_store_by_provider_mock) get_split_rule_mock = MagicMock() monkeypatch.setattr( util.stores, 'get_split_rule', get_split_rule_mock) expected_row = ( resolution, content_type, theatrical_release_date, home_video_release_date, vod_start_of_window, revenue, upc, dbo_ignored, date, transactions, transaction_type_id, paid, format_id, country_id, expected_split_amount, expected_split_id) res = util.calculate_single_row(est_dates, dbo, row) # Assertion get_store_by_provider_mock.assert_called_with(operator_id) get_split_rule_mock.assert_not_called() assert res == expected_row def test_calculate_split(monkeypatch): """Test Calculate split.""" est_date = '2016-08-15' dbo = 1000 raw_transactions = ['f', 'o', 'o'] expected_result = [ (1, 2, 3), (4, 5, 6), (7, 8, 9)] calculate_single_row_mock = MagicMock(side_effect=expected_result) monkeypatch.setattr( util, 'calculate_single_row', calculate_single_row_mock) res = [i for i in util.calculate_split(est_date, dbo, raw_transactions)] assert calculate_single_row_mock.call_count is 3 assert res == expected_result def test_load_temp_table(monkeypatch): """Test load_temp_table.""" temp_table_name = 'tmp' calculated_transactions = ('foo',) insert_q = Mock(**{'format.return_value': 'SELECT query'}) # Mocking cursor_mock = MagicMock() context = MagicMock() context.__enter__ = Mock(return_value=(cursor_mock, MagicMock())) monkeypatch.setattr( util.datastore, 'context', MagicMock(return_value=context)) util.load_temp_table( insert_q, temp_table_name, calculated_transactions) # Assertion cursor_mock.executemany.assert_called_with( 'SELECT query', calculated_transactions) def test_check_upcs(): """Test check_upcs.""" upcs = ['1', '2', '121233'] util.check_upcs(upcs) def test_check_upcs_invalid(): """Test check_upcs with invalid upcs.""" upcs = ['456789', 'abcd', '123456'] with raises(AssertionError): util.check_upcs(upcs) assert True