"""Test for data loading module.""" import os import boto3 from flexmock import flexmock import moto import pytest from sqlalchemy import text from availability_etl import config from availability_etl import constants from availability_etl import load from availability_etl.connectors import snowflake_db from tests import conftest @pytest.mark.parametrize( 'data_dicts', ( (dict(a=1, b=2), dict()), (), ) ) def test_load(data_dicts): """Test main data loading function.""" (flexmock(load) .should_receive('save_csv') .with_args(data_dicts) .once() .ordered()) mocked_funcs = ( 'upload_csv_to_s3', 'load_into_snowflake_temp_table', 'merge_snowflake_temp_and_dest_tables', ) for func in mocked_funcs: flexmock(load).should_receive(func).once().ordered() load.load(data_dicts) @pytest.fixture def aws_credentials(): """Mocked AWS Credentials for moto.""" os.environ['AWS_ACCESS_KEY_ID'] = 'testing' os.environ['AWS_SECRET_ACCESS_KEY'] = 'testing' @moto.mock_s3 def test_upload_csv_to_s3(tmpdir, aws_credentials): """Test uploaded file is under expected path and has expected content.""" flexmock(config).should_receive('S3_BUCKET_NAME').and_return('bucket') bucket = config.S3_BUCKET_NAME local_filename = 'file.csv' expected_contents = 'Lancelot,Galahad,Arthur' remote_filename = '/'.join((constants.S3_PATH_PREFIX, local_filename)) conn = boto3.resource('s3', region_name='us-east-1') conn.create_bucket(Bucket=bucket) with tmpdir.as_cwd(): with open(local_filename, 'w') as f: f.write(expected_contents) load.upload_csv_to_s3(local_filename) contents = conn.Object( config.S3_BUCKET_NAME, remote_filename).get()['Body'].read().decode() assert contents == expected_contents @pytest.mark.parametrize( 'data_dicts, expected_file_contents', ( ((), ''), ( (dict(zip(constants.CSV_COLUMNS, constants.CSV_COLUMNS)),), ','.join(constants.CSV_COLUMNS) + '\n', ), ) ) def test_save_csv(data_dicts, expected_file_contents, tmpdir): """Test local csv file is under expected path and has expected content.""" filename = constants.CSV_FILE_NAME with tmpdir.as_cwd(): expected_file_path = os.path.join(os.getcwd(), filename) assert load.save_csv(data_dicts) == expected_file_path with open(expected_file_path, 'r') as expected_file: assert expected_file.read() == expected_file_contents def test_load_into_snowflake_temp_table(get_fake_db_connection): """Test loading data into Snowflake table calls expected functions.""" conn = get_fake_db_connection() (flexmock(snowflake_db) .should_receive('get_snowflake_connection') .and_return(conn) .ordered()) (flexmock(conftest) .should_receive('fake_connection_execute') .with_args(constants.SQL_SF_TRUNCATE_TEMP) .once() .ordered()) (flexmock(conftest) .should_receive('fake_connection_execute') .with_args(type(text(''))) .once() .ordered()) load.load_into_snowflake_temp_table() def test_merge_snowflake_temp_and_dest_tables(get_fake_db_connection): """Test Snowflake merge operation is executed with expected parameters.""" conn = get_fake_db_connection() (flexmock(snowflake_db) .should_receive('get_snowflake_connection') .and_return(conn) .ordered()) (flexmock(conftest) .should_receive('fake_connection_execute') .with_args(constants.SQL_MERGE_TEMP_TO_DEST) .once() .ordered()) load.merge_snowflake_temp_and_dest_tables()