import codecs import unittest from unittest.mock import patch, Mock import boto3 import pytest from moto import mock_s3 from requests import Response from s3_uploader.command import get_copier from s3_uploader.copier.url_to_s3 import Copier as UrlToS3Copier from s3_uploader.copier.url_to_s3_decompress import Copier as UrlToS3DecompressCopier from s3_uploader.exceptions import InvalidArgumentError @mock_s3 class UrlToS3CopierTest(unittest.TestCase): def setUp(self): boto3.setup_default_session() self.conn = boto3.resource('s3', region_name='us-east-1') self.bucket = 'dev-sme-stream-transfer' self.conn.create_bucket(Bucket=self.bucket) self.expected = 'Some bytes data' data = bytes(self.expected, 'utf-8') self.expected_compressed = codecs.encode(data, "zlib") def test_get_copier_from_url_to_s3(self): copier = get_copier( Mock(), 'https://youtu.be/EngW7tLk6R8', 's3://dev-sme-stream-transfer/10MB.zip', None ) self.assertIsInstance(copier, UrlToS3Copier) copier = get_copier( Mock(), 'https://ws.spotify.com/analytics/api/resource/path?oauth_token=my_token', 's3://my-s3-bucket/spotify/resource.gz', None ) self.assertIsInstance(copier, UrlToS3Copier) @patch('requests.head') @patch('requests.get') def test_copier_from_url_to_s3(self, r_get_mock, r_head_mock): val = Response() val._content = self.expected_compressed val.headers = { 'Content-Length': 5, } r_get_mock.return_value = val r_head_mock.return_value = val copier = get_copier( Mock(), 'http://ipv4.download.thinkbroadband.com/initial_test_file.gz', f's3://{self.bucket}/test_file.gz', None ) copier.copy() actual_compressed = self.conn.Object('dev-sme-stream-transfer', 'test_file.gz').get()['Body'].read() self.assertEqual(actual_compressed, self.expected_compressed) @patch('requests.head') @patch('requests.get') def test_copier_from_url_to_s3_decompressed(self, r_get_mock, r_head_mock): val = Response() val._content = self.expected_compressed val._content_consumed = True val.headers = { 'Content-Length': 5, } r_get_mock.return_value = val r_head_mock.return_value = val copier = get_copier( Mock(), 'http://ipv4.download.thinkbroadband.com/initial_test_file.gz', f's3://{self.bucket}/test_file.gz', f's3://{self.bucket}/test_file.txt', ) copier.copy() actual_compressed = self.conn.Object('dev-sme-stream-transfer', 'test_file.gz').get()['Body'].read() self.assertEqual(actual_compressed, self.expected_compressed) actual_decompressed = self.conn.Object('dev-sme-stream-transfer', 'test_file.txt').get()['Body'].read().decode("utf-8") self.assertEqual(actual_decompressed, self.expected) def test_invalid_copy(self): with pytest.raises(InvalidArgumentError): assert get_copier( Mock(), 'http://ipv4.download.thinkbroadband.com/10MB.zip', 'http://ipv4.download.thinkbroadband.com/10MB.zip', None ) with pytest.raises(InvalidArgumentError): assert get_copier( Mock(), 's3://dev-sme-stream-transfer/10MB.zip', 'http://ipv4.download.thinkbroadband.com/10MB.zip', None ) def test_get_decompress_copier_and_gzip_to_s3(self): copier = get_copier( Mock(), 'https://ws.spotify.com/analytics/api/resource/path?oauth_token=my_token', 's3://my-s3-bucket/spotify/resource.gz', 's3://my-s3-bucket/spotify/resource.csv' ) self.assertIsInstance(copier, UrlToS3DecompressCopier)