# pylint: disable=protected-access,unused-argument import awswrangler as wr import pandas as pd import pytest from slz_spotify_charts_scraper.const import FileExtension from slz_spotify_charts_scraper.entities import Config, FilePath from slz_spotify_charts_scraper.service import S3Service, SpotifyChartsService @pytest.mark.parametrize('target_bucket', ['decompressed_bucket', 'corrupted_bucket']) def test_move_files_between_buckets( s3_client, s3_service: S3Service, config_test: Config, target_bucket: str ): source_bucket: str = config_test.quarantine_bucket source_key = 'spotify/charts_daily_regional/v1/report_date=2021-08-01/' \ 'report_licensor=sme/charts_daily_regional_20210801_us.parquet' s3_client.put_object(Bucket=source_bucket, Key=source_key, Body='test') # Assert initial conditions. target_bucket_items = wr.s3.list_objects(path=f's3://{target_bucket}') target_path_string = f's3://{target_bucket}/{source_key}' assert target_path_string not in target_bucket_items # Assert FilePath instances are constructed correctly. source_path = FilePath.from_str(f's3://{source_bucket}/{source_key}') target_path = FilePath.from_str(target_path_string) assert source_path.to_str() == f's3://{source_bucket}/{source_key}' assert target_path.to_str() == target_path_string s3_service.move_file(source_path=source_path, target_path=target_path) # Assert move operation result. source_bucket_items = wr.s3.list_objects(path=f's3://{config_test.quarantine_bucket}') target_bucket_items = wr.s3.list_objects(path=f's3://{target_bucket}') assert f's3://{config_test.quarantine_bucket}/{source_key}' not in source_bucket_items assert f's3://{target_bucket}/{source_key}' in target_bucket_items @pytest.mark.parametrize( 'target_bucket, market', [('decompressed_bucket', 'us'), ('corrupted_bucket', 'us')] ) def test_upload_raw_data_to_bucket( s3_client, spotify_service_stateless: SpotifyChartsService, s3_service: S3Service, config_test: Config, target_bucket: str, market: str, ): raw_data = b'bytes_raw_data_dummy_content' file_path = spotify_service_stateless._get_path_to_file( market=market, bucket=target_bucket, payload=spotify_service_stateless._payload, extension=FileExtension.HTML, ) s3_service.upload_bytes(file_path, raw_data) uploaded_files = wr.s3.list_objects(path=file_path.get_prefix()) assert file_path.to_str() in uploaded_files @pytest.mark.parametrize( 'target_bucket,market', [('decompressed_bucket', 'us'), ('corrupted_bucket', 'us')] ) def test_upload_dataframe_parquet_to_bucket( s3_client, spotify_service_stateless: SpotifyChartsService, s3_service: S3Service, config_test: Config, target_bucket: str, market: str, ): data_frame = pd.DataFrame({'test_id': [1, 2, 3, 4, 5]}) file_path = spotify_service_stateless._get_path_to_file( market=market, bucket=target_bucket, payload=spotify_service_stateless._payload, extension=FileExtension.PARQUET, ) s3_service.upload_dataframe(file_path, data_frame) uploaded_files = wr.s3.list_objects(path=file_path.get_prefix()) assert file_path.to_str() in uploaded_files def test_delete_objects( s3_client, spotify_service_stateless: SpotifyChartsService, s3_service: S3Service, config_test: Config, ): raw_data = b'bytes_raw_data_dummy_content' target_bucket = config_test.quarantine_bucket file_path = spotify_service_stateless._get_path_to_file( market='us', bucket=target_bucket, payload=spotify_service_stateless._payload, extension=FileExtension.HTML, ) s3_service.upload_bytes(file_path, raw_data) initially_uploaded = wr.s3.list_objects(path=file_path.get_prefix()) assert file_path.to_str() in initially_uploaded s3_service.delete_objects(paths=[file_path]) after_removal = wr.s3.list_objects(path=file_path.get_prefix()) assert file_path.to_str() not in after_removal def test_get_file_size(s3_client, s3_service: S3Service, config_test: Config): char_length = 100 source_bucket = config_test.quarantine_bucket source_key = 'spotify/charts_daily_regional/v1/report_date=2021-08-01/' \ 'report_licensor=sme/charts_daily_regional_20210801_us.parquet' s3_client.put_object(Bucket=source_bucket, Key=source_key, Body='#' * char_length) source_path = f's3://{source_bucket}/{source_key}' file_path = FilePath.from_str(source_path) assert source_path == file_path.to_str() size = s3_service.get_file_size(file_path) assert size == char_length