# pylint: disable=protected-access import gzip import unittest from unittest.mock import Mock, patch import boto3 import smart_open from moto import mock_s3 from parameterized import parameterized from exp_archive_lambda.entities import ArchiveParams, Input from exp_archive_lambda.service import Service @mock_s3 # pylint: disable=too-many-instance-attributes class ServiceArchiveFileTestCase(unittest.TestCase): def setUp(self): boto3.setup_default_session() self.conn = boto3.resource('s3', region_name='us-east-1') self.client = boto3.client('s3', region_name='us-east-1') self.bucket = 'dev-sme-stream-transfer' self.conn.create_bucket(Bucket=self.bucket) self.expected = 'header1\nheader2\ntext1\ntext2\ntext3\nfooter1\nfooter2\n' data = bytes(self.expected, 'utf-8') self.initial_s3_path = f's3://{self.bucket}/initial_test_file.txt' self.compressed_s3_path = f's3://{self.bucket}/compressed_test_file.txt.gz' with smart_open.open(self.initial_s3_path, 'wb') as fout: fout.write(data) self.input = Input( source_path='', dest_bucket='', archive_params=ArchiveParams(drop_from_head=0, drop_from_tail=0), disassemble_content_status_id=123, content_name='test_file.txt', ) self.service = Service( Mock(), self.input, self.client, ) @parameterized.expand([ (0, 0, 'header1\nheader2\ntext1\ntext2\ntext3\nfooter1\nfooter2\n'), (1, 0, 'header2\ntext1\ntext2\ntext3\nfooter1\nfooter2\n'), (1, 1, 'header2\ntext1\ntext2\ntext3\nfooter1\n'), (2, 2, 'text1\ntext2\ntext3\n'), ]) def test_copy_with_drop(self, drop_from_head, drop_from_tail, result): self.service._payload.archive_params.drop_from_head = drop_from_head self.service._payload.archive_params.drop_from_tail = drop_from_tail self.service._archive_with_drop(self.initial_s3_path, self.compressed_s3_path) actual = self.conn.Object(self.bucket, 'compressed_test_file.txt.gz').get()['Body'].read() actual_decompressed = gzip.decompress(actual).decode('utf-8') self.assertEqual(actual_decompressed, result) def test_copy_without_drop(self): self.service._payload.archive_params.drop_from_head = 0 self.service._payload.archive_params.drop_from_tail = 0 self.service._archive_with_drop(self.initial_s3_path, self.compressed_s3_path) actual = self.conn.Object(self.bucket, 'compressed_test_file.txt.gz').get()['Body'].read() actual_decompressed = gzip.decompress(actual).decode('utf-8') self.assertEqual(actual_decompressed, self.expected) class ServiceUtilsTestCase(unittest.TestCase): def setUp(self): self.input = Input( source_path='', dest_bucket='', archive_params=ArchiveParams(drop_from_head=0, drop_from_tail=0), disassemble_content_status_id=123, content_name='test_file.txt', ) self.service = Service(Mock(), self.input, Mock()) def test_parse_path(self): bucket, key = self.service._parse_path( 's3://bucket-test/test_folder/test/initial_test_file.txt' ) self.assertEqual(bucket, 'bucket-test') self.assertEqual(key, 'test_folder/test/initial_test_file.txt') @parameterized.expand([ ( 's3://bucket-test/test_folder/test/initial_test_file.txt', 'new-bucket', 123, 's3://new-bucket/test_folder/test/initial_test_file/123/initial_test_file-' '20ae0578-be4d-48bb-bb0a-5bc81d1aa93e.txt.gz', ), ( 's3://bucket-test/test_folder/test/initial_test_file.ex1.ex2.txt', 'new-bucket', 123, 's3://new-bucket/test_folder/test/initial_test_file.ex1.ex2/123/initial_test_file-' '20ae0578-be4d-48bb-bb0a-5bc81d1aa93e.ex1.ex2.txt.gz' ), ( 's3://dev-sme-data-decompressed/vevo/sony_standard_sales/v1/' 'report_date=2020-07-09/report_licensor=sme/SonyStandardSales_20200709.tsv', 'new-bucket', 123, 's3://new-bucket/vevo/sony_standard_sales/v1/' 'report_date=2020-07-09/report_licensor=sme/SonyStandardSales_20200709/123/' 'SonyStandardSales_20200709-20ae0578-be4d-48bb-bb0a-5bc81d1aa93e.tsv.gz', ), ]) @patch('uuid.uuid4', lambda: '20ae0578-be4d-48bb-bb0a-5bc81d1aa93e') def test_get_dest_path(self, initial_path, initial_bucket, disassemble_cs_id, expected): path = self.service._get_dest_path(initial_path, initial_bucket, disassemble_cs_id) self.assertEqual(path, expected)