"""Lambda test module.""" import boto3 from unittest.mock import MagicMock from unittest import mock from moto import mock_s3 import os import pytest from constants import general import index @pytest.fixture def event(): """Fixture that returns lambda event.""" return { 'Records': [ { 'eventVersion': '2.0', 'eventTime': '1970-01-01T00:00:00.000Z', 'requestParameters': { 'sourceIPAddress': '127.0.0.1' }, 's3': { 'configurationId': 'testConfigRule', 'object': { 'eTag': 'b9120b8ebaa8ad198fd0b31ea2bf99d4', 'sequencer': '0A1B2C3D4E5F678901', 'key': 'ppl_stmt/input/test.csv', 'size': 1024 }, 'bucket': { 'arn': 'arn:aws:s3:::test-s3-bucket', 'name': 'test-s3-bucket', 'ownerIdentity': { 'principalId': 'EXAMPLE' } }, 's3SchemaVersion': '1.0' }, 'responseElements': { 'x-amz-id-2': 'EXAMPLE123', 'x-amz-request-id': 'EXAMPLE123456789' }, 'awsRegion': 'us-east-1', 'eventName': 'ObjectCreated:Put', 'userIdentity': { 'principalId': 'EXAMPLE' }, 'eventSource': 'aws:s3' } ] } csv_content = [['105618854', 'The Orchard Enterprises', 'Public Performance & Broadcasting', '2016-12'], ['105618855', 'The Orchard Enterprises', 'Public Performance & Broadcasting', '2016-12'], ['105618856', 'The Orchard Enterprises', 'New Media', '2016-12']] updated_csv_content = [['105618854', 'The Orchard Enterprises', 'Public Performance & Broadcasting', '2016-12', '12-Dec-2017', '12-Dec-2017', 'ppl_jun2018.csv', '1024'], ['105618855', 'The Orchard Enterprises', 'Public Performance & Broadcasting', '2016-12', '12-Dec-2017', '12-Dec-2017', 'ppl_jun2018.csv', '1024'], ['105618856', 'The Orchard Enterprises', 'New Media', '2016-12', '12-Dec-2017', '12-Dec-2017', 'ppl_jun2018.csv', '1024']] def seed_s3(event, key_name='ppl_monthyear.csv'): """Seed s3 with content for test cases.""" bucket_name = event['Records'][0]['s3']['bucket']['name'] key_contents = """PPL MEMBER ID,MEMBER NAME,REVENUE TYPE,ALLOCATION PERIOD 105618854,The Orchard Enterprises,Public Performance & Broadcasting,2016-12 105618855,The Orchard Enterprises,Public Performance & Broadcasting,2016-12 105618856,The Orchard Enterprises,New Media,2016-12""" s3 = boto3.resource('s3', region_name='eu-west-1') bucket = s3.create_bucket(Bucket=bucket_name) batchfile = s3.Object(bucket_name, key_name) batchfile.put(Body=key_contents) return bucket, batchfile @mock.patch('index.ingest_file') @mock.patch('index.write_to_s3') @mock.patch('index.update_csv_data') @mock.patch('index.read_s3_file') @mock.patch('index.SimpleIngestManager') @mock.patch('index.get_snowflake_private_key') def test_handler( mock_key, mock_simpleingestmanager, mock_reads3file, mock_updatecsvdata, mock_writes3file, mock_ingestfile, event): """Test Lambda entry point.""" key_name = event['Records'][0]['s3']['object']['key'] bucket_name = event['Records'][0]['s3']['bucket']['name'] filename = os.path.basename(key_name) filesize = event['Records'][0]['s3']['object']['size'] mock_key.return_value = 'secret' index.handler(event, None) mock_key.assert_called() mock_reads3file.assert_called_with( bucket_name, key_name, event['Records'][0]) sanitized_csv = [row[:17] for row in mock_reads3file.return_value[1:0]] mock_updatecsvdata.assert_called_with(sanitized_csv, filename, filesize) mock_writes3file.assert_called_with( bucket_name, key_name, mock_updatecsvdata.return_value) mock_ingestfile.assert_called_with( event, mock_simpleingestmanager.return_value, bucket_name, key_name) @mock.patch('index.boto3.session.Session') def test_get_snowflake_private_key(mock_session): """Test function which fetch private key from AWS.""" secret_name = 'test_secret_name' mock_conn = MagicMock() mock_client = MagicMock() secret_manager_dict = {} secret_manager_dict['Name'] = secret_name secret_value = 'secret_value' secret_manager_dict['SecretString'] = '{{"{}":"{}"}}'.format( secret_name, secret_value) mock_client.get_secret_value.return_value = secret_manager_dict mock_conn.client.return_value = mock_client mock_session.return_value = mock_conn result = index.get_snowflake_private_key() assert result == secret_manager_dict['SecretString'] @mock_s3 def test_read_s3_file(event): """Test function to read S3 file from bucket.""" key_name = 'ppl_jun2018.csv' bucket_name = event['Records'][0]['s3']['bucket']['name'] seed_s3(event, key_name) record = event['Records'][0] all_rows = index.read_s3_file( bucket_name, key_name, record) assert all_rows[1:] == csv_content @mock_s3 def test_write_s3_file(event): """Test function to write S3 file into output staging file.""" key_name = 'ppl_jun2018.csv' output_key_name = '{}/{}'.format(general.PPL_STMT_OUTPUT_PATH, key_name) bucket_name = event['Records'][0]['s3']['bucket']['name'] record = event['Records'][0] seed_s3(event, key_name) index.write_to_s3(bucket_name, key_name, updated_csv_content) allrows = index.read_s3_file( bucket_name, output_key_name, record) assert allrows == updated_csv_content @mock_s3 @mock.patch('index.StagedFile') def test_ingest_file(mock_snowflakeingest, event): """Test function to ingest S3 file into Snowflake.""" key_name = 'ppl_jun2018.csv' bucket_name = event['Records'][0]['s3']['bucket']['name'] seed_s3(event, key_name) ingest_manager = MagicMock() ingest_manager.ingest_files.return_value = { 'responseCode': general.SUCCESS_CODE} resp = index.ingest_file(event, ingest_manager, bucket_name, key_name) assert resp == {'status': 'OK'}