"""Unit tests for the tasks of the GfK ingestion workflow.""" from unittest.mock import ANY from unittest.mock import call from unittest.mock import MagicMock from unittest.mock import patch from freezegun import freeze_time from garcon_contrib.dynamo_feed_status import garcon_feed_status from feed_ingestion.flows.gfk import config from feed_ingestion.flows.gfk import tasks @freeze_time('2000-01-01') @patch('feed_ingestion.flows.gfk.tasks.garcon_feed_status') def test_bootstrap(feed_status_mock, monkeypatch): """Check that bootstrap returns expected results.""" context = { 'activity': MagicMock(), 'date': None, 'reload': False, 'dw_config': { 'schema': 'production' }} source_files_dict = {'files': [ {'file_name': 'Membran-weekly_200001.txt'}]} # expected response expected = { 'date': '2000-01-01', 'feed_name': config.feed_name, 'secrets_path': config.secrets_path, 's3_archive_path': 's3://dev-cucumbers/GFK/archives/2000-01-01/', 'staging_raw_table': config.snowflake_table_names['staging_raw'], 'source_files_dict': source_files_dict} # check response reply = tasks.bootstrap(**context) assert reply == expected # pass explict date context = { 'activity': MagicMock(), 'date': '2001-01-01', 'reload': False, 'dw_config': { 'schema': 'production' }} expected = { 'date': '2001-01-01', 'feed_name': config.feed_name, 'secrets_path': config.secrets_path, 's3_archive_path': 's3://dev-cucumbers/GFK/archives/2001-01-01/', 'staging_raw_table': config.snowflake_table_names['staging_raw'], 'source_files_dict': ANY} reply = tasks.bootstrap(**context) assert reply == expected @freeze_time('2000-01-01') @patch( 'feed_ingestion.flows.gfk.tasks.garcon_feed_status.get_overall_status') def test_bootstrap_ingested(mock_feed_status): """Check that flow exits if status is INGESTED.""" mock_feed_status.return_value = garcon_feed_status.STATUS_INGESTED response = tasks.bootstrap(MagicMock(), None, False) assert response == { 'message': 'gfk is already ingested for 2000-01-01', 'stop': True} @patch('feed_ingestion.tasks.reporter_tasks.garcon_feed_status.' 'set_overall_status') @patch('feed_ingestion.flows.gfk.tasks._get_sftp_connection') @patch('feed_ingestion.flows.gfk.tasks.os.remove') @patch('builtins.open') @patch('feed_ingestion.flows.gfk.tasks._get_latest_file') @patch('feed_ingestion.flows.gfk.tasks.s3') def test_fetch_from_drop_location( s3_mock, latest_file, open_mock, os_mock, ftp_mock, mock_set_overall_status): """Test custom fetch_from_drop_location task.""" # case 1 ftp_creds = { 'host': 'testhost', 'username': 'testusername', 'password': 'testpassword', 'path': 'testdir'} feed_name = config.feed_name date = '2018-08-24' latest_file.return_value = { 'filename': 'Membran-weekly_2018-08-24_blabla.txt', 'size': 'size' } activity_mock = MagicMock() result = tasks.fetch_from_drop_location( activity_mock, feed_name, date, 's3://somebucket/somepath/', ftp_creds) ftp_mock.assert_has_calls([ call(ftp_creds), call().listdir_attr('/testdir'), call().get( '/testdir/Membran-weekly_2018-08-24_blabla.txt', 'Membran-weekly_2018-08-24_blabla.txt'), call().close() ]) assert s3_mock.upload_to_s3.call_args_list == [ call( file_path='Membran-weekly_2018-08-24_blabla.txt', bucket_name='dev-cucumbers', object_key='somepath/Membran-weekly_2018-08-24_blabla.txt'), ] # case 2 s3_mock.upload_to_s3.reset_mock() filename_mock = MagicMock() filename_mock.return_value = [ 'Membran-weekly_2018-08-24_blabla.txt', 'Membran-weekly_2018-08-24_blabla1.txt', 'Membran-weekly_2018-08-25_blabla.txt'] ftp_conn_mock = MagicMock() ftp_conn_mock.listdir_attr.return_value = filename_mock ftp_conn_mock.sendcmd.return_value = [ '213 20180825111501.784', '213 20180825101501.784', '213 20180826101501.784'] ftp_mock.return_value = ftp_conn_mock feed_name = config.feed_name result = tasks.fetch_from_drop_location( activity_mock, feed_name, date, 's3://somebucket/somepath/', ftp_creds) assert s3_mock.upload_to_s3.call_args_list == [ call( file_path='Membran-weekly_2018-08-24_blabla.txt', bucket_name='dev-cucumbers', object_key='somepath/Membran-weekly_2018-08-24_blabla.txt'), ] assert os_mock.called assert (result['source_files_dict']['files'][0]['file_name'] == 'Membran-weekly_2018-08-24_blabla.txt')