# pylint: disable=protected-access import unittest from unittest.mock import Mock import boto3 import smart_open from moto import mock_s3 from parameterized import parameterized from exp_archive_lambda.entities import ArchiveParams, Input from exp_archive_lambda.service_copy import CopyService @mock_s3 class ServiceCopyTestCase(unittest.TestCase): def setUp(self): boto3.setup_default_session() self.conn = boto3.resource('s3', region_name='us-east-1') self.bucket = 'dev-sme-stream-transfer' self.conn.create_bucket(Bucket=self.bucket) self.expected = 'header1\nheader2\ntext1\ntext2\ntext3\nfooter1\nfooter2\n' data = bytes(self.expected, 'utf-8') initial_files = [f's3://{self.bucket}/if_1.parquet', f's3://{self.bucket}/if_2.parquet'] for init_file in initial_files: with smart_open.open(init_file, 'wb') as fout: fout.write(data) self.input = Input( source_path=','.join(initial_files), dest_bucket=self.bucket, archive_params=ArchiveParams(drop_from_head=0, drop_from_tail=0), disassemble_content_status_id=123, content_name='youtubereporting.csv.gz' ) self.service = CopyService( Mock(), self.input, ) def test_copy_ok(self): is_ok, chunks_count = self.service.copy() self.assertEqual(is_ok, True) self.assertEqual(chunks_count, 2) for _f in [ 'youtubereporting.csv/123/if_1.parquet', 'youtubereporting.csv/123/if_2.parquet' ]: actual = self.conn.Object(self.bucket, _f).get()['Body'].read().decode('utf-8') self.assertEqual(actual, self.expected) @parameterized.expand([ ( 'test_folder/test/', 'test_file.parquet', 'youtubereporting.csv.gz', 'test_folder/test/youtubereporting.csv/123/test_file.parquet' ), ( 'test_folder/', 'test_file.parquet', 'youtubereporting.csv.gz', 'test_folder/youtubereporting.csv/123/test_file.parquet' ), ( '', 'test_file.parquet', 'youtubereporting.csv.gz', 'youtubereporting.csv/123/test_file.parquet' ), ]) def test_get_dest_key(self, prefix, fname, content_name, expected_path): initial_key = f'{prefix}{fname}' result = self.service._get_dest_key( initial_key=initial_key, content_name=content_name, dcs_id=123 ) self.assertEqual(result, expected_path)