"""Lambda test module.""" import csv import os from unittest import mock from unittest.mock import MagicMock from urllib.parse import unquote_plus import boto3 from constants import general import index from moto import mock_s3 import pytest @pytest.fixture def event(): """Fixture that returns lambda event.""" key = 'soundexchange-sales-stmt/input/test_soundexchange.csv' return { 'Records': [ { 'eventVersion': '2.0', 'eventTime': '1970-01-01T00:00:00.000Z', 'requestParameters': { 'sourceIPAddress': '127.0.0.1' }, 's3': { 'configurationId': 'testConfigRule', 'object': { 'eTag': 'b9120b8ebaa8ad198fd0b31ea2bf99d4', 'sequencer': '0A1B2C3D4E5F678901', 'key': key, 'size': 1024 }, 'bucket': { 'arn': 'arn:aws:s3:::test-s3-bucket', 'name': 'test-s3-bucket', 'ownerIdentity': { 'principalId': 'EXAMPLE' } }, 's3SchemaVersion': '1.0' }, 'responseElements': { 'x-amz-id-2': 'EXAMPLE123', 'x-amz-request-id': 'EXAMPLE123456789' }, 'awsRegion': 'us-east-1', 'eventName': 'ObjectCreated:Put', 'userIdentity': { 'principalId': 'EXAMPLE' }, 'eventSource': 'aws:s3' } ] } csv_content = [['Artist Name', 'Track Name', 'Album Name', 'LABEL_NAME', 'Schedule ID', 'Catalog Number', 'Broadcast Start Date'], ['artist1', 'track1', 'album1', 'label1', '11', '12', '01-JAN-17'], ['artist2', 'track2', 'album2', 'label2', '21', '22', '01-FEB-17'], ['artist3', 'track3', 'album3', 'label3', '31', '32', '01-MAR-17']] updated_csv_content = [['Type', 'SCR Repertoire ID', 'Royalty Item SXID', 'Artist Name', 'Track Name', 'Album Name', 'Label Name', 'ISRC', 'Release UPC', 'RO Name', 'Licensee Name', 'Category of Service', 'Broadcast Start Date', 'Broadcast End Date', 'Num Performances', 'Num Spins', 'Allocation Period', 'Non Ephemeral Amount', 'Ephemeral Amount', 'Your Payment Amount', 'Third Party Name', 'Third Party Payment Amount', 'Third Party Non Ephemeral Amount', 'Third Party Ephemeral Amount', 'Artist ID', 'Label ID', 'Society Label ID', 'Pline Label Name', 'Catalog Number', 'Schedule ID', 'File Name', 'File Size'], ['', '', '', 'artist1', 'track1', 'album1', 'label1', '', '', '', '', '', '01-Jan-2017', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '12', '11', 'test_soundexchange.csv', '1024'], ['', '', '', 'artist2', 'track2', 'album2', 'label2', '', '', '', '', '', '01-Feb-2017', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '22', '21', 'test_soundexchange.csv', '1024'], ['', '', '', 'artist3', 'track3', 'album3', 'label3', '', '', '', '', '', '01-Mar-2017', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '32', '31', 'test_soundexchange.csv', '1024']] def seed_s3(event, key_name='test_soundexchange.csv'): """Seed s3 with content for test cases.""" bucket_name = event['Records'][0]['s3']['bucket']['name'] key_contents = ("""Artist Name,Track Name,Album Name,LABEL_NAME,Schedule ID,Catalog Number,Broadcast Start Date""" + # noqa """\n""" + """artist1,track1,album1,label1,11,12,01-JAN-17 artist2,track2,album2,label2,21,22,01-FEB-17 artist3,track3,album3,label3,31,32,01-MAR-17""") s3 = boto3.resource('s3', region_name='eu-west-1') bucket = s3.create_bucket(Bucket=bucket_name) batchfile = s3.Object(bucket_name, key_name) batchfile.put(Body=key_contents) return bucket, batchfile @mock.patch('index.ingest_file') @mock.patch('index.write_to_s3') @mock.patch('index.SimpleIngestManager') @mock.patch('index.get_snowflake_private_key') def test_handler( mock_key, mock_simpleingestmanager, mock_writes3file, mock_ingestfile, event): """Test Lambda entry point.""" key_name = event['Records'][0]['s3']['object']['key'] bucket_name = event['Records'][0]['s3']['bucket']['name'] filename = os.path.basename(key_name) filesize = event['Records'][0]['s3']['object']['size'] mock_key.return_value = 'secret' index.handler(event, None) mock_key.assert_called() mock_writes3file.assert_called_with( bucket_name, key_name, filename, filesize) mock_ingestfile.assert_called_with( event, mock_simpleingestmanager.return_value, bucket_name, key_name) @mock.patch('index.boto3.session.Session') def test_get_snowflake_private_key(mock_session): """Test function which fetch private key from AWS.""" secret_name = 'test_secret_name' mock_conn = MagicMock() mock_client = MagicMock() secret_manager_dict = {} secret_manager_dict['Name'] = secret_name secret_value = 'secret_value' secret_manager_dict['SecretString'] = '{{"{}":"{}"}}'.format( secret_name, secret_value) mock_client.get_secret_value.return_value = secret_manager_dict mock_conn.client.return_value = mock_client mock_session.return_value = mock_conn result = index.get_snowflake_private_key() assert result == secret_manager_dict['SecretString'] @mock_s3 def test_write_s3_file(event): """Test function to write S3 file into output staging file.""" key_name = 'test_soundexchange.csv' output_key_name = '{}/{}'.format( general.SOUNDEXCHANGE_STMT_OUTPUT_PATH, key_name) bucket_name = event['Records'][0]['s3']['bucket']['name'] record = event['Records'][0] seed_s3(event, key_name) filename = os.path.basename(key_name) filesize = record['s3']['object']['size'] index.write_to_s3( bucket_name, key_name, filename, filesize) s3 = boto3.resource('s3') obj = s3.Object(bucket_name, unquote_plus(output_key_name)) lines = obj.get()['Body'].read().decode().splitlines(True) reader = csv.reader(lines) allrows = [row for row in reader] assert allrows == updated_csv_content @mock_s3 @mock.patch('index.StagedFile') def test_ingest_file(mock_snowflakeingest, event): """Test function to ingest S3 file into Snowflake.""" key_name = 'test_soundexchange.csv' bucket_name = event['Records'][0]['s3']['bucket']['name'] seed_s3(event, key_name) ingest_manager = MagicMock() ingest_manager.ingest_files.return_value = { 'responseCode': general.SUCCESS_CODE} resp = index.ingest_file(event, ingest_manager, bucket_name, key_name) assert resp == {'status': 'OK'}