"""Unit tests for tasks of Spotify Marketshare Workflow.""" from unittest.mock import MagicMock from unittest.mock import patch from garcon_contrib.dynamo_feed_status import garcon_feed_status import pytest from feed_ingestion.flows.spotify_marketshare import config from feed_ingestion.flows.spotify_marketshare import tasks _date = '2018-05-01' @pytest.fixture def expected_bootstrap_response(): """Response for bootstrap task.""" file_patterns = ['spotify-revshare-for-theorchard-20185.txt', 'spotify-legend-for-theorchard-20185.txt'] reports = { 'legend': { 'file_pattern': 'spotify-legend-for-theorchard-20185.txt', 'temp_table_name': 'temp_staging_raw_spotify_market_share_legend_20180501'}, 'revshare': { 'file_pattern': 'spotify-revshare-for-theorchard-20185.txt', 'temp_table_name': 'temp_staging_raw_spotify_market_share_revshare_20180501'}} return { 'feed_name': config.feed_name, 'secrets_path': config.secrets_path, 'date': _date, 's3_archive_path': 's3://dev-cucumbers/SpotifyMarketShare/archives/2018-05/', 's3_download_path': 's3://dev-feed-drop/ftp/SpotifyMarketShare/', 'file_patterns': file_patterns, 'reports': reports, 'staging_raw_table': 'staging_raw_spotify_market_share' } def test_bootstrap( expected_bootstrap_response): """Test bootstrap task.""" result = tasks.bootstrap( activity=MagicMock(), date=_date, reload=None) assert result == expected_bootstrap_response @pytest.fixture() def ftp_conf_mock(): """Mock ftp configuration.""" return { 'host': 'spotify_host', 'username': 'spotify_username', 'password': 'spotify_password', 'port': 22, 'path': '/path/to/spotify' } @pytest.fixture def mock_delete_status(): """Yield delete status.""" delete_status_path = ( 'feed_ingestion.flows.spotify_marketshare.tasks.' 'garcon_feed_status.delete_status') with patch(delete_status_path) as delete_status: yield delete_status @pytest.fixture def mock_task_status(): """Yield task_status.""" path = 'feed_ingestion.flows.spotify_marketshare.tasks.task_status' with patch(path) as task_status: yield task_status @pytest.fixture def mock_set_overall_status(): """Yield overall status.""" overall_status_path = ( 'feed_ingestion.flows.spotify_marketshare.tasks.' 'garcon_feed_status.set_overall_status') with patch(overall_status_path) as overall_status: yield overall_status @pytest.fixture def mock_s3utils(): """Yield s3utils.""" path = 'feed_ingestion.flows.spotify_marketshare.tasks.s3utils' with patch(path) as m_s3utils: yield m_s3utils def test_check_files_on_s3_if_there_are_new_files( mock_s3utils, mock_task_status, mock_delete_status, mock_set_overall_status): """Test check_files_on_s3.""" mock_s3utils.get_list_of_files_and_directories.return_value = [ 'US test_pattern_us.csv', 'DE test_pattern_de.csv'] mock_task_status.get_values.return_value = [] expected_response = dict( source_files_dict={ 'files': [ {'file_name': 'US test_pattern_us.csv'}, {'file_name': 'DE test_pattern_de.csv'} ]}) result = tasks.check_files_on_s3( activity=MagicMock(), feed_name='feed_name', date='2016-01-01', s3_download_path='test_path', file_patterns=['test_pattern']) mock_set_overall_status.assert_not_called() mock_delete_status.assert_called_once_with('feed_name', '2016-01-01') assert result == expected_response def test_check_files_on_s3_if_there_are_no_files( mock_s3utils, mock_task_status, mock_delete_status, mock_set_overall_status): """Test check_files_on_s3.""" mock_s3utils.get_list_of_files_and_directories.return_value = [] mock_task_status.get_values.return_value = [] expected_response = {'stop': True} result = tasks.check_files_on_s3( activity=MagicMock(), feed_name=config.feed_name, date=_date, s3_download_path='test_path', file_patterns=['test_pattern']) mock_set_overall_status.assert_called_with( config.feed_name, _date, garcon_feed_status.STATUS_NOT_AVAILABLE) mock_delete_status.assert_not_called() assert result == expected_response @pytest.fixture def mock_executor_context(): """Yield executor context.""" sf_executor_class_path = ( 'feed_ingestion.flows.spotify_marketshare.tasks.SpotifyMarketshareSF') with patch(sf_executor_class_path) as sf_executor: mock_executor_context = sf_executor.return_value.__enter__.return_value yield mock_executor_context def test_check_files_on_s3_if_there_are_no_new_files( mock_s3utils, mock_task_status, mock_delete_status, mock_set_overall_status): """Test check_files_on_ftp if there are no new files.""" mock_s3utils.get_list_of_files_and_directories.return_value = [ 'US test_pattern_us.csv', 'DE test_pattern_de.csv'] mock_task_status.get_values.return_value = [ 'US test_pattern_us.csv', 'DE test_pattern_de.csv'] result = tasks.check_files_on_s3( activity=MagicMock(), feed_name='feed_name', date='2016-01-01', s3_download_path='test_path', file_patterns=['test_pattern']) mock_set_overall_status.assert_not_called() mock_delete_status.assert_not_called() assert result == {'stop': True} def test_check_files_on_s3_need_all_patterns( mock_s3utils, mock_task_status, mock_delete_status, mock_set_overall_status): """Test check_files_on_s3 if there is only one type of file on ftp.""" mock_s3utils.get_list_of_files_and_directories.return_value = [ 'US test_pattern_us.csv', 'DE test_pattern_de.csv'] mock_task_status.get_values.return_value = [] result = tasks.check_files_on_s3( activity=MagicMock(), feed_name=config.feed_name, date=_date, s3_download_path='test_path', file_patterns=['test_pattern', 'test_pattern2']) mock_set_overall_status.assert_called_with( config.feed_name, _date, garcon_feed_status.STATUS_NOT_AVAILABLE) mock_delete_status.assert_not_called() assert result == {'stop': True} def test_drop_temp_table(mock_executor_context): """Test drop_temp_table task.""" tasks.drop_temp_table(MagicMock(), 'test_table') (mock_executor_context.drop_table.assert_called_with( 'test_table'))