"""Test generators.""" from unittest.mock import MagicMock import pytest from yt_conflict_elasticsearch.flows.elasticsearch_export import config from yt_conflict_elasticsearch.flows.elasticsearch_export import generators TEST_INDEX = 'test_index' @pytest.fixture def conflicts(): """Conflicts from SF fixture.""" return [('a665a45920422f9d417e4867efdc4fb8' 'a04a1f3fff1fa07e998e86f7f7a27ae3',), ('f8339a2baa293b3902b3b3d9d561938a' '4385c262148193013b776be4e15e6f3a',)] @pytest.fixture def expected_items(): """Generate ES items.""" return iter([{ '_index': TEST_INDEX, '_op_type': 'delete', '_id': 'a665a45920422f9d417e4867efdc4fb8' 'a04a1f3fff1fa07e998e86f7f7a27ae3' }, { '_index': TEST_INDEX, '_op_type': 'delete', '_id': 'f8339a2baa293b3902b3b3d9d561938a' '4385c262148193013b776be4e15e6f3a' }]) @pytest.fixture def conflict_ids_for_territories(): """Test conflict ids for territories.""" return iter([[31487132]]) @pytest.fixture def conflict_modified(): """Test modified conflict.""" conflicts_modified = { '_index': config.OS_CONFLICTS_INDEX_NAME, '_op_type': 'update', '_id': 'EWAvpWcBLzAzl7XpJ9Dv', 'doc': { 'territories': [ { 'conflict_id': 4971125, 'code': 'EC', 'continent_name': 'South America', 'name': 'Ecuador', }, { 'conflict_id': 5033762, 'code': 'AR', 'continent_name': 'South America', 'name': 'Argentina' } ] } } return iter([conflicts_modified]) @pytest.fixture def conflict_documents(): """Get a sample document-form conflicts.""" conflict_data = [ { 'territory_standard': 'ISO_3166_1_2016', 'conflict_date': '2017-04-26', 'resolved_datetime': None, 'tuid': 111911, 'isrc': 'USA370376686', 'product_id': 24434, 'conflicting_owner': 'rumblefish_cdbaby', 'track_name': 'Amazing Grace (remix)', 'vendor_id': 36, 'track_artists': ['Just Plain Darin', 'Mama Mia'], 'display_upc': '669910788622', 'territories': [ { 'conflict_id': 31487132, 'code': 'BQ', 'continent_name': 'North America', 'name': 'Bonaire' } ], 'product_name': 'Back To The Basics', 'subaccount_name': 'Hot Potato Productions', 'subaccount_id': 15, 'daily_average_views': 0, 'status': 'NEW', 'views_in_conflict': 2 }] return ({ '_index': config.OS_CONFLICTS_INDEX_NAME, '_source': c, } for c in conflict_data) @pytest.fixture() def es_insertion_iterator(): """Test ES insert iterator.""" return MagicMock() @pytest.fixture() def conflict_ids_iterator(): """Test conflict ID iteragor.""" return iter([[1, 2, 3, 4]]) def test_conflicts_to_remove_os(conflicts, expected_items): """Test generator yields valid items.""" for item in generators.conflicts_to_remove_os(conflicts, TEST_INDEX): assert item == next(expected_items) def test_get_unindexed_conflict_ids_by_line( save_gzipped_file_with_json_conflicts, conflict_ids_for_territories): """Test generator yield a list of conflict_ids from territories.""" test_file_path = 'tests/test_files/conflicts.json.gz' for item in generators.get_unindexed_conflict_ids_by_line(test_file_path): assert item == next(conflict_ids_for_territories) def test_get_es_ids_to_update_territories_os( save_gzipped_file_with_json_conflicts, conflict_modified): """Test generator yield a list of modified conflict_ids.""" test_file_path = 'tests/test_files/conflicts.json.gz' for item in generators.get_es_ids_to_update_territories_os(test_file_path): assert item == next(conflict_modified) def test_get_conflict_data_to_populate_os( save_gzipped_file_with_json_conflicts, conflict_documents): """Test get_conflict_data_to_populate.""" test_file_path = 'tests/test_files/conflicts.json.gz' for item in generators.get_conflict_data_to_populate_os( test_file_path, config.OS_CONFLICTS_INDEX_NAME): assert item == next(conflict_documents) def test_get_batch_to_insert_to_sf_temp( mock_config, conflict_ids_iterator, es_insertion_iterator): """Test get_batch_to_insert_to_sf_temp.""" expected_items = [ { 'conflict_id': 1, 'es_id': 'somehash1' }, { 'conflict_id': 2, 'es_id': 'somehash1' }, { 'conflict_id': 3, 'es_id': 'somehash1' }, { 'conflict_id': 4, 'es_id': 'somehash1' } ] es_insertion_iterator.__next__.side_effect = [ (True, {'index': {'_id': 'somehash1'}}) ] batch_iterator = generators.get_batch_to_insert_to_sf_temp( conflict_ids_iterator, es_insertion_iterator) items = [] for batch in batch_iterator: assert len(batch) == config.SF_INSERT_BATCH_SIZE items.extend(batch) assert items == expected_items