import gzip import json import unittest.mock import boto3 import pytest from moto import mock_s3 from slz_downloader.dsp.linkfire import Client from slz_downloader.entities import Job @pytest.mark.parametrize( 'report_type, context, source_file_prefix', [ ( 'linkfire_raw_data', 'Linkfire_raw_data', 'Sony_Music_Entertainment_Linkfire_raw_data_2021-03-08_2021-03-08' ), ( 'applemusic_raw_data', 'applemusic_raw_data', 'applemusic_sony-music-entertainment_2021-03-08-2021-03-08' ) ] ) @unittest.mock.patch('slz_downloader.dsp.linkfire.get_secret') @mock_s3 def test_linkfire_downloading( get_secret, report_type, context, source_file_prefix, aws_credentials, params ): job = Job.from_dict( { 'uow_id': 'linkfire-20210308-sme-Linkfire_raw_data-v1_0', 'dsp': 'linkfire', 'report_type': report_type, 'version': 'v1_0', 'report_date': '2021-03-08', 'licensor': 'sme', 'extension': 'csv', 'context': context, } ) boto3.setup_default_session() conn = boto3.resource('s3', region_name='us-east-1') for bucket in ['bucket-archive-quarantine', 'bucket-decompressed-quarantine', 'source_test']: conn.create_bucket(Bucket=bucket) # Client initialization logger = unittest.mock.Mock() client = Client(logger=logger) # Mock secrets get_secret.return_value = json.dumps({ 'BUCKET': 'source_test', }) client.configure(params) expected_file_content = 'Some bytes data' data = bytes(expected_file_content, 'utf-8') expected_compressed = gzip.compress(data, 9) source_file = conn.Object( 'source_test', f'raw_data/6ee94b8a-4c47-11e6-9fd0-066c3e7a8751/{source_file_prefix}.csv.gz' ) source_file.put(Body=expected_compressed) client.download(job, chunk_size=1) # read data from mocked S3 buckets path = f'linkfire/{report_type}/v1_0/report_date=2021-03-08/report_licensor=sme' actual_decompressed = conn.Object( 'bucket-decompressed-quarantine', f'{path}/{source_file_prefix}.csv' ).get()['Body'].read() actual_compressed = conn.Object( 'bucket-archive-quarantine', f'{path}/{source_file_prefix}.csv.gz' ).get()['Body'].read() assert expected_file_content == actual_decompressed.decode('utf8') assert expected_file_content == gzip.decompress(actual_compressed).decode('utf8')