import codecs import io import unittest import unittest.mock import gzip import boto3 import moto from botocore.exceptions import ClientError from s3_uploader.copier.smart_stream_to_s3_decompress import Copier as SmartStreamToS3DecompressCopier class SmartStreamToS3CopierTest(unittest.TestCase): @moto.mock_s3 def test_copier_from_stream_to_s3_decompressed_gz(self): boto3.setup_default_session() conn = boto3.resource('s3', region_name='us-east-1') conn.create_bucket(Bucket='dev-sme-stream-transfer') expected = 'Some bytes data' data = bytes(expected, 'utf-8') expected_compressed = gzip.compress(data, 9) dest = { 'bucket': 'dev-sme-stream-transfer', 'key': 'test_file.gz' } decompressed_dest = { 'bucket': 'dev-sme-stream-transfer', 'key': 'test_file.txt' } copier = SmartStreamToS3DecompressCopier( unittest.mock.Mock(), io.BytesIO(expected_compressed), dest, decompressed_dest, ) total = copier.copy() self.assertEqual(total, len(expected_compressed)) actual_compressed = conn.Object(dest['bucket'], dest['key']).get()['Body'].read() self.assertEqual(actual_compressed, expected_compressed) actual_decompressed = conn.Object(decompressed_dest['bucket'], decompressed_dest['key']).get()['Body'].read().decode("utf-8") self.assertEqual(actual_decompressed, expected) @moto.mock_s3 def test_copier_from_stream_to_s3_decompress_gz_only_decompress(self): boto3.setup_default_session() conn = boto3.resource('s3', region_name='us-east-1') conn.create_bucket(Bucket='dev-sme-stream-transfer') expected = 'Some bytes data' data = bytes(expected, 'utf-8') expected_compressed = gzip.compress(data, 9) dest = { 'bucket': 'dev-sme-stream-transfer', 'key': 'test_file.gz' } decompressed_dest = { 'bucket': 'dev-sme-stream-transfer', 'key': 'test_file.txt' } copier = SmartStreamToS3DecompressCopier( unittest.mock.Mock(), io.BytesIO(expected_compressed), dest, decompressed_dest, decompress_only=True ) total = copier.copy() self.assertEqual(total, len(expected_compressed)) self.assertRaises(ClientError, conn.Object(dest['bucket'], dest['key']).get) actual_decompressed = conn.Object(decompressed_dest['bucket'], decompressed_dest['key']).get()['Body'].read().decode("utf-8") self.assertEqual(actual_decompressed, expected)