"""Integration test for Distribution Fee ETL.""" from datetime import date from decimal import Decimal from itertools import chain from unittest.mock import patch from garcon_contrib.dynamo_feed_status import garcon_feed_status import pytest from flows import datastore from flows.distribution_fee import config as flow_config from integration_tests.flows.distribution_fee import config from integration_tests.flows.distribution_fee import helpers from integration_tests.flows.distribution_fee import queries from integration_tests.flows.distribution_fee.fixtures import joined_data @pytest.fixture(params=[ flow_config.TABLE_DS_REVENUE_CABLE, flow_config.TABLE_DS_REVENUE_DIGITAL, flow_config.TABLE_DS_REVENUE_THEATRICAL]) def revenue_table(request): """Revenue table to examine.""" return request.param def _is_close(a, b, err=Decimal(1e-02)): """Check if two Decimals are close enough.""" return abs(a - b) <= err def _infer_effective_split(entry, target_table): """Infer split to apply based on defied priorities. Args: entry (dict): entry dict as defined in joined_data.py. target_table (str): target revenue table Returns: tuple: split type, split value. """ # @TODO: implement DMS split ter_table = flow_config.TABLE_AR_VENDOR_TER_CONTRACT if ter_table in entry and ( entry[ter_table]['country_id'] == entry[target_table]['country_id']): return 'territory', entry[ter_table]['territory_split'] else: return 'regular', entry[ flow_config.TABLE_AR_VENDOR_CONTRACT]['digital_split'] def _upcs_with_split(split_type): """Filter upcs with specified split from the entire set. Args: split_type (str): split type to filter entries by. Returns: list(tuple(str, float, float, str)): list of tuples: (upc, amount_value, split_value, table_name). """ revenue_tables = [ (flow_config.TABLE_DS_REVENUE_CABLE, 'orchard_amount'), (flow_config.TABLE_DS_REVENUE_DIGITAL, 'amount'), (flow_config.TABLE_DS_REVENUE_THEATRICAL, 'orchard_amount')] upcs = [] for entry in joined_data.entries: for table_name, amount_column in revenue_tables: if table_name in entry: break else: raise Exception('No known table found in entry: %s' % entry) entry_split_type, entry_split_value = _infer_effective_split( entry, table_name) if entry_split_type != split_type: continue upcs.append( (entry[table_name]['upc'], entry[table_name][amount_column], entry_split_value, table_name)) return upcs def _check_split(upc, amount, split, table_name): """Check that client_value is close enough to expected one.""" client_amount, *_ = datastore.query(""" SELECT client_amount FROM {db_name}.{table_name} WHERE upc = %s;""".format( db_name=config.TEST_DS_DB_NAME, table_name=table_name), (upc,)).fetchone() return _is_close(client_amount, Decimal(amount * split)) @pytest.fixture(params=_upcs_with_split('territory')) def upc_with_territory_split(request): """Fixture returning upc with effective territory split.""" return request.param @pytest.fixture(params=_upcs_with_split('regular')) def upc_with_regular_split(request): """Fixture returning upc with effective regular split.""" return request.param @pytest.fixture(scope='module') def upcs(): """Set upcs fixture.""" return [ '889845260455', '190374485784', '191018166793', '889176688676', '889176849459', '190374817103', '190374502856', '190374917803', '191018032753', '190374851541', '889845026730'] @pytest.fixture(scope='module') def db_tables(): """Create and populate all the tables that the ETL uses as source.""" tables = list(chain( queries.TABLES_DS, queries.TABLES_AR)) for create_stmt, drop_stmt in queries.DATABASES: datastore.execute(drop_stmt) datastore.execute(create_stmt) for table_name, (create_stmt, ins_stmt) in tables: datastore.execute(create_stmt) if ins_stmt: data = helpers.read_test_data(table_name) datastore.executemany(ins_stmt, data) yield for _, drop_stmt in queries.DATABASES: datastore.execute(drop_stmt) @pytest.fixture(scope='module') def completed_etl_run(db_tables, upcs): """Run Distribution fee etl.""" with patch.dict('os.environ', config.MOCK_ENV_DB_CONFIG): helpers.execute_flow(upcs) def test_client_amount_set(completed_etl_run, revenue_table): """Test that client_amount column was set at least for a single entry.""" count, *_ = datastore.query(""" SELECT COUNT(*) FROM {db_name}.{table_name} WHERE client_amount IS NOT NULL;""".format( db_name=config.TEST_DS_DB_NAME, table_name=revenue_table)).fetchone() assert count > 0 def test_territory_split(completed_etl_run, upc_with_territory_split): """Test that if defined territory split takes precedence over regular.""" assert _check_split(*upc_with_territory_split) def test_regular_split(completed_etl_run, upc_with_regular_split): """Test that regular split is applied if nothing else is defined.""" assert _check_split(*upc_with_regular_split) def test_dynamo_status(completed_etl_run): """Test if dynamo status was set.""" today_str = date.today().strftime('%Y-%m-%d') assert garcon_feed_status.get_overall_status( flow_config.SWF_WORKFLOW_NAME, today_str) == 'INGESTED'