import codecs import unittest from unittest import mock import boto3 import smart_open from moto import mock_s3 from s3_uploader.command import get_copier from s3_uploader.copier.s3_to_s3 import Copier as S3ToS3Copier from s3_uploader.copier.s3_to_s3_decompress import Copier as S3ToS3DecompressCopier @mock_s3 class S3ToS3CopierTest(unittest.TestCase): def setUp(self): boto3.setup_default_session() self.conn = boto3.resource('s3', region_name='us-east-1') self.bucket = 'dev-sme-stream-transfer' self.conn.create_bucket(Bucket=self.bucket) expected = 'Some bytes data' data = bytes(expected, 'utf-8') self.expected_compressed = codecs.encode(data, "zlib") self.initial_s3_path = f's3://{self.bucket}/initial_test_file.gz' with smart_open.open(self.initial_s3_path, 'wb', ignore_ext=True) as fout: fout.write(self.expected_compressed) def test_get_copier_from_s3_to_s3(self): copier = get_copier( mock.Mock(), 's3://dev-sme-stream-transfer/10MB.zip', 's3://dev-sme-stream-transfer/10_2MB.zip', None ) self.assertIsInstance(copier, S3ToS3Copier) def test_copier_from_s3_to_s3(self): copier = get_copier( mock.Mock(), self.initial_s3_path, f's3://{self.bucket}/test_file.gz', None ) self.assertIsInstance(copier, S3ToS3Copier) copier.copy() actual_compressed = self.conn.Object('dev-sme-stream-transfer', 'test_file.gz').get()['Body'].read() self.assertEqual(actual_compressed, self.expected_compressed) def test_get_decompress_copier_from_s3_to_s3(self): copier = get_copier( mock.Mock(), 's3://my-s3-bucket/spotify/resource.gz', 's3://my-s3-bucket/spotify/resource.gz', 's3://my-s3-bucket/spotify/resource.csv' ) self.assertIsInstance(copier, S3ToS3DecompressCopier) def test_copier_from_s3_to_s3_decompressed(self): copier = get_copier( mock.Mock(), self.initial_s3_path, f's3://{self.bucket}/test_file.gz', f's3://{self.bucket}/test_file.txt' ) self.assertIsInstance(copier, S3ToS3DecompressCopier) self.assertRaises(NotImplementedError, copier.copy)