from unittest.mock import MagicMock from unittest.mock import patch import pytest import smart_open from processing_accounting.flows.accounting_statement_export import setting from processing_accounting.flows.accounting_statement_export import tasks from processing_accounting.util import db as db_util from processing_accounting.util import dynamodb as dynamodb_util def test_bootstrap(monkeypatch): mock_activity = MagicMock() monkeypatch.setattr( mock_activity, 'info', MagicMock(return_value=None)) with pytest.raises(AssertionError): tasks.bootstrap(mock_activity, '205', None, None, None, None) with pytest.raises(AssertionError): tasks.bootstrap(mock_activity, '202,203,204', None, None, None, None) mock_config = setting.config test_bucket = setting.report_bucket with patch.dict(setting.config, mock_config, clear=True): resp = tasks.bootstrap( mock_activity, '205', 'label', None, None, None) assert resp.get('schematized_file_path') == ( 's3://{}/schematized_files/205_label_all_all_month'.format( test_bucket)) assert resp.get('payment_interval') == 'month' expected_track_artists_table = 'dim_track_artists_aggregated_temp' actual_track_artist_table = resp.get('temp_track_artists_table_name') assert actual_track_artist_table == expected_track_artists_table def test_hashFor(monkeypatch): expected = '2f4a8dbd4cdc82139c47d0df78b540ac' actual = tasks._hashFor('test') assert expected == actual @patch( 'processing_accounting.flows.accounting_statement_export.tasks.' 'SqlGenerator') @patch( 'processing_accounting.flows.accounting_statement_export.tasks.' 'SnowflakeSQLExecutor') def test_get_unload_query(mock_executor, mock_sql_generator, monkeypatch): mock_activity = MagicMock() monkeypatch.setattr( mock_activity, 'info', MagicMock(return_value=None)) mock_sql_generator_obj = MagicMock() monkeypatch.setattr( mock_sql_generator_obj, 'get_sql', MagicMock(return_value="'")) mock_sql_generator.return_value = mock_sql_generator_obj executor = MagicMock(return_value={'COUNT(*)': 333}) mock_executor.return_value.__enter__.return_value.fetchone = executor resp = tasks.get_unload_query( mock_activity, '202,203,204', 'label', None, None, None) assert mock_sql_generator_obj.get_sql.called assert resp['unload_query'] == "'" @patch( 'processing_accounting.flows.accounting_statement_export.tasks.' 'SqlGenerator', autospec=True) @patch( 'processing_accounting.flows.accounting_statement_export.tasks.' 'SnowflakeSQLExecutor') def test_get_unload_query_no_data( mock_executor, mock_sql_generator, monkeypatch): mock_activity = MagicMock() monkeypatch.setattr( mock_activity, 'info', MagicMock(return_value=None)) mock_sql_generator_obj = MagicMock() monkeypatch.setattr( mock_sql_generator_obj, 'get_sql', MagicMock(return_value="'")) mock_sql_generator.return_value = mock_sql_generator_obj executor = MagicMock(return_value={'COUNT(*)': 0}) mock_executor.return_value.__enter__.return_value.fetchone = executor resp = tasks.get_unload_query( mock_activity, '202,203,204', 'label', None, None, None) assert mock_sql_generator_obj.get_sql.called assert resp == {'stop': True, 'message': 'No data found.'} def test_generate_hql(monkeypatch): mock_file_pointer = MagicMock() mock_smart_open_obj = MagicMock() monkeypatch.setattr( smart_open, 'smart_open', MagicMock( return_value=mock_smart_open_obj)) monkeypatch.setattr( mock_file_pointer, 'write', MagicMock(return_value='test')) monkeypatch.setattr( mock_smart_open_obj, '__enter__', MagicMock( return_value=mock_file_pointer)) monkeypatch.setattr( mock_smart_open_obj, '__exit__', MagicMock( return_value=None)) mock_config = { 'emr': { 'hql': { 'setting': 'test_setting', 'drop_accounting_statement_export': 'test_drop{key}', 'create_accounting_statement_export': 'test_create_table{key}', 'drop_accounting_statement_export_avro': 'test_drop_avro{key}', 'create_avro_table': 'test_create_avro{key}', 'write_avro_files': 'test_write_avro{key}' }, 'hql_path': 'test_hql_path' } } mock_activity = MagicMock() monkeypatch.setattr( mock_activity, 'info', MagicMock(return_value=None)) with patch.dict(setting.config, mock_config, clear=True): tasks.generate_hql(mock_activity, 'test_key') assert mock_file_pointer.write.called mock_file_pointer.write.assert_any_call('test_setting') mock_file_pointer.write.assert_any_call('test_droptest_key') mock_file_pointer.write.assert_any_call('test_create_tabletest_key') mock_file_pointer.write.assert_any_call('test_drop_avrotest_key') mock_file_pointer.write.assert_any_call('test_create_avrotest_key') mock_file_pointer.write.assert_any_call('test_write_avrotest_key') @patch( 'processing_accounting.flows.accounting_statement_export.tasks.' 'SqlGenerator') def test_update_status(mock_sql_generator, monkeypatch): mock_activity = MagicMock() monkeypatch.setattr( mock_activity, 'info', MagicMock(return_value=None)) mock_sql_generator_obj = MagicMock() mock_db_result = [ { 'user_id_type': '2343L', 'status': 'generating', 's3_path': 's3://test_bucket/test_key' } ] monkeypatch.setattr( dynamodb_util, 'set_status', MagicMock(return_value=None)) monkeypatch.setattr( db_util, 'snowflake_query', MagicMock(return_value=mock_db_result)) monkeypatch.setattr( mock_sql_generator_obj, 'get_sql_for_bulk_status_update', MagicMock(return_value="'")) mock_sql_generator.return_value = mock_sql_generator_obj tasks.update_status( mock_activity, '204,205,206', 'label', 'month', 'all', 'generating') dynamodb_util.set_status.assert_any_call( '2343L', '204,205,206__all__AVRO__en_US', 'generating', period_ids='204,205,206', s3_path='s3://test_bucket/test_key') tasks.update_status( mock_activity, '205', 'label', 'month', '3333', 'generating') dynamodb_util.set_status.assert_any_call( '3333L', '205__all__AVRO__en_US', 'generating', period_ids='205')