"""Unit tests for the tasks of the FeatureFmFacebook ingestion workflow.""" from unittest.mock import MagicMock from unittest.mock import patch from freezegun import freeze_time from garcon_contrib.dynamo_feed_status import garcon_feed_status from feed_ingestion.flows.feature_fm_facebook import config from feed_ingestion.flows.feature_fm_facebook import tasks @freeze_time('2020-06-01') @patch('feed_ingestion.flows.feature_fm_facebook.tasks.garcon_feed_status') @patch('feed_ingestion.flows.feature_fm_facebook.tasks.s3') def test_bootstrap(s3_mock, feed_status_mock, monkeypatch): """Check that bootstrap returns expected results.""" context = { 'activity': MagicMock(), 'reload': False, 'date': '2020-06-01', 'report_type': 'daily,monthly', 'dw_config': { 'schema': 'production' }} mock_dir_path = 's3://dev-cucumbers/FeatureFmAdSpend/' mock_key = mock_dir_path + 'feature-fm-ad-campaign-2020_06_01_blah.csv' s3_mock.get_key_size.return_value = 1240 s3_mock.get_list_of_files_and_directories.return_value = [mock_key] source_files_dict = {'files': [ { 'file_name': 'feature-fm-ad-campaign-2020_06_01_blah.csv', 'found': True, 'file_size': 1240}, # extra file for monthly ingest { 'file_name': 'feature-fm-ad-campaign-2020_06_01_blah.csv', 'found': True, 'file_size': 1240} ]} clean_name = 'feature-fm-ad-campaign-2020_06_01_blah_clean.csv' preprocess_dict = { 'feature-fm-ad-campaign-2020_06_01_blah.csv': clean_name } # expected response expected = { 'feed_name': config.feed_name, 'secrets_path': config.secrets_path, 'date': '2020-06-01', 'staging_raw_table': config.snowflake_table_names['staging_raw'], 's3_dir_path': 's3://dev-cucumbers/FeatureFmAdSpend/', 'source_files_dict': source_files_dict, 'preprocess_file_name_map': preprocess_dict } # check response reply = tasks.bootstrap(**context) assert reply == expected # pass explict date context = { 'activity': MagicMock(), 'reload': False, 'date': '2020-06-01', 'report_type': 'daily, monthly', 'dw_config': { 'schema': 'production' }} expected = { 'date': '2020-06-01', 'feed_name': config.feed_name, 'secrets_path': config.secrets_path, 's3_dir_path': 's3://dev-cucumbers/FeatureFmAdSpend/', 'staging_raw_table': config.snowflake_table_names['staging_raw'], 'source_files_dict': source_files_dict, 'preprocess_file_name_map': preprocess_dict} reply = tasks.bootstrap(**context) assert reply == expected @freeze_time('2000-01-01') @patch( 'feed_ingestion.flows.feature_fm_facebook.tasks.garcon_feed_status' '.get_overall_status') def test_bootstrap_ingested(mock_feed_status): """Check that flow exits if status is INGESTED.""" mock_feed_status.return_value = garcon_feed_status.STATUS_INGESTED response = tasks.bootstrap(MagicMock(), None, False, 'daily') assert response == { 'message': 'feature_fm_facebook is already ingested for 2000-01-01', 'stop': True} @patch('feed_ingestion.flows.feature_fm_facebook.tasks.get_sf_config') @patch('feed_ingestion.flows.feature_fm_facebook.tasks.FeatureFmFacebook') def test_deduplicate_fact_table(mock_sf_executor_class, mock_get_sf_config): """Test deduplicate_fact_table.""" mock_config = {'db': 'DB', 'schema': 'SCHEMA'} mock_get_sf_config.return_value = mock_config mock_executor = MagicMock() mock_sf_executor_class.return_value.__enter__.return_value = mock_executor tasks.deduplicate_fact_table(MagicMock(), '2000-01-01') mock_get_sf_config.assert_called_once_with(config.feed_name) mock_sf_executor_class.assert_called_once_with(mock_config) mock_executor.delete_from_fact_table.assert_called_once_with('2000-01-01')