import os import tempfile import unittest import boto3 import botocore import pandas as pd from moto import mock_s3 from service.utils.aws_connectors import s3_to_pandas MY_BUCKET = "fansifter_mock_bucket" @mock_s3 class TestS3(unittest.TestCase): def setUp(self): client = boto3.client( "s3", region_name="eu-west-1", aws_access_key_id="fake_access_key", aws_secret_access_key="fake_secret_key", ) try: s3 = boto3.resource( "s3", region_name="eu-west-1", aws_access_key_id="fake_access_key", aws_secret_access_key="fake_secret_key", ) s3.meta.client.head_bucket(Bucket=MY_BUCKET) except botocore.exceptions.ClientError: pass else: err = f"{MY_BUCKET} should not exist." raise EnvironmentError(err) client.create_bucket(Bucket=MY_BUCKET) current_dir = os.path.dirname(__file__) fixtures_dir = os.path.join(current_dir, "s3_mock_files") _upload_fixtures(MY_BUCKET, fixtures_dir) def tearDown(self): s3 = boto3.resource( "s3", region_name="eu-west-1", aws_access_key_id="fake_access_key", aws_secret_access_key="fake_secret_key", ) bucket = s3.Bucket(MY_BUCKET) for key in bucket.objects.all(): key.delete() bucket.delete() def test_s3_to_pandas(self): """Test if s3_to_pandas indeed returns pandas dataframe""" with tempfile.TemporaryDirectory(): df = s3_to_pandas(MY_BUCKET, "data.csv", header=None, file_format="csv") self.assertTrue(isinstance(df, pd.DataFrame)) def _upload_fixtures(bucket: str, fixtures_dir: str) -> None: client = boto3.client("s3") fixtures_paths = [ os.path.join(path, filename) for path, _, files in os.walk(fixtures_dir) for filename in files ] for path in fixtures_paths: key = os.path.relpath(path, fixtures_dir) client.upload_file(Filename=path, Bucket=bucket, Key=key) if __name__ == "__main__": unittest.main()