"""Amazon Music Generators.""" from unittest.mock import patch from garcon_contrib.dynamo_feed_status import garcon_feed_status import pytest from feed_ingestion.flows.amazon_music import generators @pytest.fixture def context_amazon_unlimited(): """Context for amazon_music_test flow.""" return { 'bootstrap_feed.feed_name': 'amazon_music_test', 'bootstrap_feed.date': '2017-08-01', 'bootstrap_feed.licensor': 'theorchard', 'bootstrap_feed.report_name': 'test', 'bootstrap_feed.s3_drop_location': 's3://dev-feed-drop/feed-drop/AmazonTest/', 'bootstrap_feed.archive_bucket': 's3://dev-cucumbers/AmazonTest/archives/2017-08-01/', 'bootstrap_feed.clean_path': 's3://dev-cucumbers/AmazonTest/archives/2017-08-01/clean/', 'bootstrap_feed.expected_files': [ 'ORCA_US_20170801_Daily_Playlist_Report.txt.zip', 'ORCA_US_20170801_Daily_User_Report.txt.zip'], 'bootstrap_feed.countries': ['US', 'AT'], 'bootstrap_feed.snowflake_error_limit': 1, 'bootstrap_feed.snowflake_error_on_column_count_mismatch': 'true'} @patch('feed_ingestion.flows.amazon_music.generators.' 'garcon_feed_status.get_status') def test_generator_returns_keys( mock_get_status, context_amazon_unlimited): """Generator returns source and destination keys.""" mock_get_status.return_value = garcon_feed_status.STATUS_NOT_INGESTED generator = generators.drop_files_generator(context_amazon_unlimited) expected_result = [{ 'source_key_name': 'feed-drop/AmazonTest/' 'ORCA_US_20170801_Daily_Playlist_Report.txt.zip', 'destination_key_name': 'AmazonTest/archives/2017-08-01/' 'ORCA_US_20170801_Daily_Playlist_Report.txt.zip'}, { 'source_key_name': 'feed-drop/AmazonTest/' 'ORCA_US_20170801_Daily_User_Report.txt.zip', 'destination_key_name': 'AmazonTest/archives/2017-08-01/' 'ORCA_US_20170801_Daily_User_Report.txt.zip'}] assert list(generator) == expected_result @patch('feed_ingestion.flows.amazon_music.generators.' 'garcon_feed_status.get_status') def test_generator_should_not_yield_downloaded_files( mock_get_feed_status, context_amazon_unlimited): """Generator does not yield downloaded files.""" mock_get_feed_status.side_effect = [ garcon_feed_status.STATUS_DOWNLOADED, garcon_feed_status.STATUS_NOT_INGESTED] generator = generators.drop_files_generator(context_amazon_unlimited) expected_result = [{ 'source_key_name': 'feed-drop/AmazonTest/' 'ORCA_US_20170801_Daily_User_Report.txt.zip', 'destination_key_name': 'AmazonTest/archives/2017-08-01/' 'ORCA_US_20170801_Daily_User_Report.txt.zip'}] assert list(generator) == expected_result def test_temp_staging_tables_generator(context_amazon_unlimited, mock_config): """Test temp_staging_tables_generator.""" generator = generators.temp_staging_tables_generator( context_amazon_unlimited) expected_result = [ {'key_dir': 's3://dev-cucumbers/AmazonTest/archives/2017-08-01' '/clean/ORCA_US_20170801_Daily_Playlist_Report.txt.gz', 'kwargs': {'date_for_sqlloader': '2017-08-01', 'error_on_column_count_mismatch': 'true', 'report_type': 'Playlist'}, 'temp_staging_raw_table': 'amazon_music_test_daily_theorchard' '_Playlist_report_ORCA_US_20170801'}, {'key_dir': 's3://dev-cucumbers/AmazonTest/archives/2017-08-01' '/clean/ORCA_US_20170801_Daily_Activity_Report.txt.gz', 'kwargs': {'date_for_sqlloader': '2017-08-01', 'error_on_column_count_mismatch': 'true', 'report_type': 'Activity'}, 'temp_staging_raw_table': 'amazon_music_test_daily_theorchard' '_Activity_report_ORCA_US_20170801'}, {'key_dir': 's3://dev-cucumbers/AmazonTest/archives/2017-08-01' '/clean/ORCA_US_20170801_Daily_User_Report.txt.gz', 'kwargs': {'date_for_sqlloader': '2017-08-01', 'error_on_column_count_mismatch': 'true', 'report_type': 'User'}, 'temp_staging_raw_table': 'amazon_music_test_daily_theorchard' '_User_report_ORCA_US_20170801'}, {'key_dir': 's3://dev-cucumbers/AmazonTest/archives/2017-08-01' '/clean/ORCA_AT_20170801_Daily_Playlist_Report.txt.gz', 'kwargs': {'date_for_sqlloader': '2017-08-01', 'error_on_column_count_mismatch': 'true', 'report_type': 'Playlist'}, 'temp_staging_raw_table': 'amazon_music_test_daily_theorchard' '_Playlist_report_ORCA_AT_20170801'}, {'key_dir': 's3://dev-cucumbers/AmazonTest/archives/2017-08-01' '/clean/ORCA_AT_20170801_Daily_Activity_Report.txt.gz', 'kwargs': {'date_for_sqlloader': '2017-08-01', 'error_on_column_count_mismatch': 'true', 'report_type': 'Activity'}, 'temp_staging_raw_table': 'amazon_music_test_daily_theorchard' '_Activity_report_ORCA_AT_20170801'}, {'key_dir': 's3://dev-cucumbers/AmazonTest/archives/2017-08-01' '/clean/ORCA_AT_20170801_Daily_User_Report.txt.gz', 'kwargs': {'date_for_sqlloader': '2017-08-01', 'error_on_column_count_mismatch': 'true', 'report_type': 'User'}, 'temp_staging_raw_table': 'amazon_music_test_daily_theorchard' '_User_report_ORCA_AT_20170801'}, {'key_dir': 's3://dev-cucumbers/AmazonTest/archives/2017-08-01' '/clean/ORED_US_20170801_Daily_Playlist_Report.txt.gz', 'kwargs': {'date_for_sqlloader': '2017-08-01', 'error_on_column_count_mismatch': 'true', 'report_type': 'Playlist'}, 'temp_staging_raw_table': 'amazon_music_test_daily_theorchard_' 'Playlist_report_ORED_US_20170801'}, {'key_dir': 's3://dev-cucumbers/AmazonTest/archives/2017-08-01' '/clean/ORED_US_20170801_Daily_Activity_Report.txt.gz', 'kwargs': {'date_for_sqlloader': '2017-08-01', 'error_on_column_count_mismatch': 'true', 'report_type': 'Activity'}, 'temp_staging_raw_table': 'amazon_music_test_daily_theorchard_' 'Activity_report_ORED_US_20170801'}, {'key_dir': 's3://dev-cucumbers/AmazonTest/archives/2017-08-01' '/clean/ORED_US_20170801_Daily_User_Report.txt.gz', 'kwargs': {'date_for_sqlloader': '2017-08-01', 'error_on_column_count_mismatch': 'true', 'report_type': 'User'}, 'temp_staging_raw_table': 'amazon_music_test_daily_theorchard_' 'User_report_ORED_US_20170801'}, {'key_dir': 's3://dev-cucumbers/AmazonTest/archives/2017-08-01' '/clean/ORED_AT_20170801_Daily_Playlist_Report.txt.gz', 'kwargs': {'date_for_sqlloader': '2017-08-01', 'error_on_column_count_mismatch': 'true', 'report_type': 'Playlist'}, 'temp_staging_raw_table': 'amazon_music_test_daily_theorchard_' 'Playlist_report_ORED_AT_20170801'}, {'key_dir': 's3://dev-cucumbers/AmazonTest/archives/2017-08-01' '/clean/ORED_AT_20170801_Daily_Activity_Report.txt.gz', 'kwargs': {'date_for_sqlloader': '2017-08-01', 'error_on_column_count_mismatch': 'true', 'report_type': 'Activity'}, 'temp_staging_raw_table': 'amazon_music_test_daily_theorchard_' 'Activity_report_ORED_AT_20170801'}, {'key_dir': 's3://dev-cucumbers/AmazonTest/archives/2017-08-01' '/clean/ORED_AT_20170801_Daily_User_Report.txt.gz', 'kwargs': {'date_for_sqlloader': '2017-08-01', 'error_on_column_count_mismatch': 'true', 'report_type': 'User'}, 'temp_staging_raw_table': 'amazon_music_test_daily_theorchard_' 'User_report_ORED_AT_20170801'} ] result = list(generator) assert result == expected_result def test_load_staging_raw_table_generator(context_amazon_unlimited, mock_config): """Test load_staging_raw_table_generator.""" generator = generators.load_staging_raw_table_generator( context_amazon_unlimited) expected_result = [ {'kwargs': { 'licensor': 'theorchard', 'date_for_sqlloader': '2017-08-01', 'temp_staging_raw_activity_table': 'amazon_music_test_daily_theorchard' '_activity_report_ORCA_US_20170801', 'temp_staging_raw_user_table': 'amazon_music_test_daily_theorchard' '_user_report_ORCA_US_20170801', 'playlist_report_filename': 'ORCA_US_20170801_Daily_Playlist_Report.txt.gz', 'user_report_filename': 'ORCA_US_20170801_Daily_User_Report.txt.gz', 'activity_report_filename': 'ORCA_US_20170801_Daily_Activity_Report.txt.gz', 'temp_staging_raw_playlist_table': 'amazon_music_test_daily_theorchard' '_playlist_report_ORCA_US_20170801'}}, {'kwargs': { 'licensor': 'theorchard', 'date_for_sqlloader': '2017-08-01', 'temp_staging_raw_activity_table': 'amazon_music_test_daily_theorchard' '_activity_report_ORCA_AT_20170801', 'temp_staging_raw_user_table': 'amazon_music_test_daily_theorchard' '_user_report_ORCA_AT_20170801', 'playlist_report_filename': 'ORCA_AT_20170801_Daily_Playlist_Report.txt.gz', 'user_report_filename': 'ORCA_AT_20170801_Daily_User_Report.txt.gz', 'activity_report_filename': 'ORCA_AT_20170801_Daily_Activity_Report.txt.gz', 'temp_staging_raw_playlist_table': 'amazon_music_test_daily_theorchard' '_playlist_report_ORCA_AT_20170801'}}, {'kwargs': { 'licensor': 'theorchard', 'date_for_sqlloader': '2017-08-01', 'temp_staging_raw_activity_table': 'amazon_music_test_daily_theorchard' '_activity_report_ORED_US_20170801', 'temp_staging_raw_user_table': 'amazon_music_test_daily_theorchard' '_user_report_ORED_US_20170801', 'playlist_report_filename': 'ORED_US_20170801_Daily_Playlist_Report.txt.gz', 'user_report_filename': 'ORED_US_20170801_Daily_User_Report.txt.gz', 'activity_report_filename': 'ORED_US_20170801_Daily_Activity_Report.txt.gz', 'temp_staging_raw_playlist_table': 'amazon_music_test_daily_theorchard' '_playlist_report_ORED_US_20170801'}}, {'kwargs': { 'licensor': 'theorchard', 'date_for_sqlloader': '2017-08-01', 'temp_staging_raw_activity_table': 'amazon_music_test_daily_theorchard' '_activity_report_ORED_AT_20170801', 'temp_staging_raw_user_table': 'amazon_music_test_daily_theorchard' '_user_report_ORED_AT_20170801', 'playlist_report_filename': 'ORED_AT_20170801_Daily_Playlist_Report.txt.gz', 'user_report_filename': 'ORED_AT_20170801_Daily_User_Report.txt.gz', 'activity_report_filename': 'ORED_AT_20170801_Daily_Activity_Report.txt.gz', 'temp_staging_raw_playlist_table': 'amazon_music_test_daily_theorchard' '_playlist_report_ORED_AT_20170801'}}] assert list(generator) == expected_result