"""S3 operations for theatrical ETL.""" from datetime import date from unittest import mock from unittest.mock import Mock from unittest.mock import patch from flows.theatrical import s3 @patch('flows.theatrical.s3.boto3') def test_find_files_paths(boto3): """Test of flows.theatrical.s3.find_files_paths function.""" boto3.resource('s3').Bucket('bucket').objects.filter = Mock( return_value=[ Mock(key='path/20161010-20161013-Theatrical.csv'), Mock(key='path/20161014-20161016-Theatrical.csv'), Mock(key='path/subfolder/20161014-20161017-Theatrical.csv'), Mock(key='path/20161014-20161015-Theatrical.csv')]) params = { 'bucket_name': 'bucket', 'prefix': 'path/', 'date_start': date(2016, 10, 10), 'date_end': date(2016, 10, 18)} result = s3.find_files(**params) assert len(result) == 2 assert result[0]['s3_path'] == 'path/20161010-20161013-Theatrical.csv' assert result[0]['date_start'] == '2016-10-10' assert result[0]['date_end'] == '2016-10-13' assert result[0]['batch_date'] == '2016-10-15' assert result[0]['filename'] == '20161010-20161013-Theatrical.csv' assert result[1]['s3_path'] == 'path/20161014-20161016-Theatrical.csv' def test__try_parse_mon_thu(): """Test of flows.theatrical.s3._try_parse_mon_thu function. File for the period from Monday till Thursday. """ bucket = 'dev' prefix = 'prefix/' s3_key = 'prefix/20161010-20161013-Theatrical.csv' result = s3._try_parse(bucket, prefix, s3_key) assert result['s3_path'] == s3_key assert result['bucket_name'] == bucket assert result['filename'] == '20161010-20161013-Theatrical.csv' assert result['date_start'] == '2016-10-10' assert result['date_end'] == '2016-10-13' assert result['batch_date'] == '2016-10-15' def test__try_parse_fri_sun(): """Test of flows.theatrical.s3._try_parse_mon_thu function. File for the period from Friday till Sunday. """ bucket = 'dev' prefix = 'prefix/' s3_key = 'prefix/20161014-20161016-Theatrical.csv' result = s3._try_parse(bucket, prefix, s3_key) assert result['s3_path'] == s3_key assert result['bucket_name'] == bucket assert result['filename'] == '20161014-20161016-Theatrical.csv' assert result['date_start'] == '2016-10-14' assert result['date_end'] == '2016-10-16' assert result['batch_date'] == '2016-10-19' def test__try_parse_with_incorrect_dates(): """Test of flows.theatrical.s3._try_parse_mon_thu function. File for the incorrect period. """ bucket = 'dev' prefix = 'prefix/' s3_key = 'prefix/20161013-20161014-Theatrical.csv' result = s3._try_parse(bucket, prefix, s3_key) assert result == {} @patch('flows.theatrical.s3.boto3') def test_download_to_csv(boto3): """Test of flows.theatrical.s3.download_to_csv function.""" bucket = 'dev' key = 'path/filename.csv' boto3.resource('s3').Object(bucket, key).get()['Body'].read = Mock( return_value=b'Hello World,b,\r1,11,\n2,22,\r\n3,33,END\r\n') header, rows = s3.download_csv(bucket, key) assert header == ['hello_world', 'b'] assert rows == [['1', '11'], ['2', '22'], ['3', '33']] def test__to_snake_case(): """Test of flows.theatrical.s3._to_snake_case function.""" assert s3._to_snake_case(' Hello World ') == 'hello_world' assert s3._to_snake_case('Many different_Words ') == 'many_different_words' @patch('flows.theatrical.s3.Key') @patch('flows.theatrical.s3.S3Connection') def test_move_files(s3_connection_class_mock, key_class_mock): """Test move_files function.""" # test data files = [{'old_bucket': 'old_bucket1', 'old_key_path': 'old_key_path1', 'new_bucket': 'new_bucket1', 'new_dir': 'new_dir1', 'file_name': 'file_name1'}, {'old_bucket': 'old_bucket2', 'old_key_path': 'old_key_path2', 'new_bucket': 'new_bucket2', 'new_dir': 'new_dir2', 'file_name': 'file_name2'}] files_count = len(files) # mocks s3_connection_mock = Mock() new_bucket_mock = Mock() old_bucket_mock = Mock() s3_connection_mock.get_bucket.side_effect = [ new_bucket_mock, old_bucket_mock] * files_count s3_connection_class_mock.return_value = s3_connection_mock new_key_mock = Mock() old_key_mock = Mock() key_class_mock.side_effect = [new_key_mock, old_key_mock] * files_count # test function call s3.move_files(files) # assertions s3_connection_class_mock.assert_called_once_with() key_class_mock.assert_has_calls( [mock.call(new_bucket_mock), mock.call(old_bucket_mock)] * files_count) new_bucket_mock.copy_key.assert_has_calls( [mock.call(new_key_mock, mock.ANY, mock.ANY)] * files_count) old_bucket_mock.delete_key.assert_has_calls( [mock.call(old_key_mock)] * files_count)