import unittest from unittest.mock import Mock import boto3 import smart_open from boto3_type_annotations.s3 import Client as S3Client from moto import mock_s3 from parameterized import parameterized from exp_s3_storage.entities import S3Input from exp_s3_storage.service import S3Service @mock_s3 class S3ServiceTestCase(unittest.TestCase): @classmethod def setUpClass(cls): cls.s3_client: S3Client = boto3.client('s3', region_name='us-east-1') def setUp(self): boto3.setup_default_session() self.conn = boto3.resource('s3', region_name='us-east-1') self.client = boto3.client('s3', region_name='us-east-1') self.bucket = 'foo-delphi-exploration' self.conn.create_bucket(Bucket=self.bucket) self.input = S3Input( content_name='foo_content_name.txt', dest_bucket=self.bucket, disassemble_content_status_id=11, source_path='s3://some_bucket/foo/bar/baz/foo_content_name.txt', ) self.service = S3Service( client=self.client, payload=self.input, logger=Mock(), ) self.s3_keys = [] def tearDown(self): for key in self.s3_keys: self.s3_client.delete_object(Bucket=self.bucket, Key=key) def _generate_files(self, file_names, subfolder, prefix='foo/bar/baz'): for file_name in file_names: key = f'{prefix}/{subfolder}/{self.input.disassemble_content_status_id}/{file_name}' s3_path = f's3://{self.bucket}/{key}' with smart_open.open(s3_path, 'wb') as fout: fout.write(b'some_file_content') self.s3_keys.append(key) @parameterized.expand( [('foo_content.txt', 'foo_content'), ('foo_c.txt.tar.gz', 'foo_c.txt.tar'), ('', '')] ) def test_list_objects__ok(self, content_name, subfolder): self.input.content_name = content_name expected_result = [ 'part-00000-foo-123.tar.gz', 'part-00001-foo-123.tar.gz', 'part-00002-foo-123.tar.gz', ] self._generate_files(expected_result, subfolder) actual_result = self.service.list_objects() self.assertEqual(expected_result, actual_result) def test_list_objects__folded_ok(self): expected_result = [ 'part-00000-foo-123.tar.gz', 'part-00001-foo-123.tar.gz', 'part-00002-foo-123.tar.gz', ] self._generate_files(expected_result, 'foo_content_name') expected_result_folded = [ 'part-00003-foo-123.tar.gz', 'part-00004-foo-123.tar.gz', ] self._generate_files( expected_result_folded, '', prefix='foo/bar/baz/foo_content_name/11/sub_folder' ) actual_result = self.service.list_objects() self.assertEqual( sorted(expected_result_folded + expected_result), sorted(actual_result), )