# pylint: disable=unused-argument,protected-access,too-many-arguments from unittest import mock import awswrangler as wr import pandas as pd import pytest from boto3_type_annotations.s3 import Client as S3Client from slz_spotify_charts_data_extender.service import S3Service, Service, VendorAPIService from .conftest import get_df_source_data, get_public_data @pytest.mark.parametrize( 'market,tracks_list', [ ( 'us', [ { 'id': '3Wrjm47oTz2sjIgck11l5e', 'name': 'Beggin\'', 'release_date': '2017-12-08', 'isrc': 'ITB001700846', 'artists': [{ 'name': 'Måneskin', 'id': '0lAWpj5szCSwM4rUMHYmrr' }], 'image_url': 'https://i.scdn.co/image/ab67616d0000b273fa0ab3a28b5c52d8a5f97045' }, { 'id': '5PjdY0CKGZdEuoNab3yDmX', 'name': 'STAY (with Justin Bieber)', 'release_date': '2021-07-23', 'isrc': 'USSM12103949', 'artists': [ { 'name': 'The Kid LAROI', 'id': '2tIP7SsRs7vjIcLrU85W8J' }, { 'name': 'Justin Bieber', 'id': '1uNFoZAHBGtllmzznpCI3s' } ], 'image_url': 'https://i.scdn.co/image/ab67616d0000b2738e6551a2944764bc8e33a960' }, { 'id': '27NovPIUIRrOZoCHxABJwK', 'name': 'INDUSTRY BABY (feat. Jack Harlow)', 'release_date': '2021-07-23', 'isrc': 'USSM12104539', 'artists': [ { 'name': 'Lil Nas X', 'id': '7jVv8c5Fj3E9VhNjxT4snq' }, { 'name': 'Jack Harlow', 'id': '2LIk90788K0zvyj2JJVwkJ' } ], 'image_url': 'https://i.scdn.co/image/ab67616d0000b273ba26678947112dff3c3158bf' } ] ), ( 'global', [ { 'id': '3Wrjm47oTz2sjIgck11l5e', 'name': 'Beggin\'', 'release_date': '2017-12-08', 'isrc': 'ITB001700846', 'artists': [{ 'name': 'Måneskin', 'id': '0lAWpj5szCSwM4rUMHYmrr' }], 'image_url': 'https://i.scdn.co/image/ab67616d0000b273fa0ab3a28b5c52d8a5f97045' }, { 'id': '5PjdY0CKGZdEuoNab3yDmX', 'name': 'STAY (with Justin Bieber)', 'release_date': '2021-07-23', 'isrc': 'USSM12103949', 'artists': [ { 'name': 'The Kid LAROI', 'id': '2tIP7SsRs7vjIcLrU85W8J' }, { 'name': 'Justin Bieber', 'id': '1uNFoZAHBGtllmzznpCI3s' } ], 'image_url': 'https://i.scdn.co/image/ab67616d0000b2738e6551a2944764bc8e33a960' }, { 'id': '27NovPIUIRrOZoCHxABJwK', 'name': 'INDUSTRY BABY (feat. Jack Harlow)', 'release_date': '2021-07-23', 'isrc': 'USSM12104539', 'artists': [ { 'name': 'Lil Nas X', 'id': '7jVv8c5Fj3E9VhNjxT4snq' }, { 'name': 'Jack Harlow', 'id': '2LIk90788K0zvyj2JJVwkJ' } ], 'image_url': 'https://i.scdn.co/image/ab67616d0000b273ba26678947112dff3c3158bf' } ] ), ('au', []) ] ) def test_get_tracks_data(vendor_api_service: VendorAPIService, market, tracks_list): vendor_api_service._send_request = mock.Mock(return_value=get_public_data(market)) data = vendor_api_service.get_tracks_data( track_ids=['5PjdY0CKGZdEuoNab3yDmX', '3Wrjm47oTz2sjIgck11l5e', '27NovPIUIRrOZoCHxABJwK'], market=market ) assert tracks_list == data data_dict = vendor_api_service.get_track_data_map(data) assert data_dict == {item['id']: item for item in tracks_list} def test__build__directory_path(s3_service: S3Service): source_path = s3_service._build_source_directory_path() target_path = s3_service._build_target_directory_path() assert source_path == 's3://decompressed_bucket/spotify/charts_daily_regional/v1/report_date=2021-08-01/report_licensor=sme' assert target_path == 's3://target_bucket/spotify/charts_daily_regional/v1/report_date=2021-08-01/test_job' def test_retrieve_files(s3_client: S3Client, s3_service: S3Service, config_test): files_to_extract = ['1.json', '2.json', '3.json'] df_json = get_df_source_data('us') data_frame = pd.DataFrame(df_json) parquet_file = data_frame.to_parquet(engine='pyarrow') for _file in files_to_extract: s3_client.put_object( Body=parquet_file, Bucket=config_test.source_bucket, Key= f'spotify/charts_daily_regional/v1/report_date=2021-08-01/report_licensor=sme/{_file}', ) extracted_files = s3_service.retrieve_files(files_to_extract) assert len(list(extracted_files)) == 3 def test_upload_df_parquet_to_bucket(s3_client: S3Client, s3_service: S3Service, config_test): df_json = get_df_source_data('us') data_frame = pd.DataFrame(df_json) file_name = 'test.py' s3_service.upload_df_parquet_to_bucket(data_frame=data_frame, file_name=file_name) bucket_files = s3_client.list_objects(Bucket=config_test.target_bucket) assert len(bucket_files['Contents']) == 1 @pytest.mark.parametrize( 'dim_track_data,dim_artist_data,link_track_artist_data', [ ( { 'job_id': { 0: 'test_job', 1: 'test_job', 2: 'test_job' }, 'name': { 0: 'Happier Than Ever', 1: 'STAY (with Justin Bieber)', 2: 'INDUSTRY BABY (feat. Jack Harlow)' }, 'spotify_track_id': { 0: '3Wrjm47oTz2sjIgck11l5e', 1: '5PjdY0CKGZdEuoNab3yDmX', 2: '27NovPIUIRrOZoCHxABJwK' }, 'spotify_country_id': { 0: 'us', 1: 'us', 2: 'us' }, 'isrc': { 0: 'ITB001700846', 1: 'USSM12103949', 2: 'USSM12104539' }, 'source_artists': { 0: 'Billie Eilish', 1: 'The Kid LAROI', 2: 'Lil Nas X' }, 'uri': { 0: 'https://open.spotify.com/track/4RVwu0g32PAqgUiJoXsdF8', 1: 'https://open.spotify.com/track/5PjdY0CKGZdEuoNab3yDmX', 2: 'https://open.spotify.com/track/27NovPIUIRrOZoCHxABJwK' }, 'release_date': { 0: '2017-12-08', 1: '2021-07-23', 2: '2021-07-23' }, 'image_url': { 0: 'https://i.scdn.co/image/ab67616d0000b273fa0ab3a28b5c52d8a5f97045', 1: 'https://i.scdn.co/image/ab67616d0000b2738e6551a2944764bc8e33a960', 2: 'https://i.scdn.co/image/ab67616d0000b273ba26678947112dff3c3158bf' } }, { 'job_id': { 0: 'test_job', 1: 'test_job', 2: 'test_job', 3: 'test_job', 4: 'test_job' }, 'spotify_artist_id': { 0: '0lAWpj5szCSwM4rUMHYmrr', 1: '2tIP7SsRs7vjIcLrU85W8J', 2: '1uNFoZAHBGtllmzznpCI3s', 3: '7jVv8c5Fj3E9VhNjxT4snq', 4: '2LIk90788K0zvyj2JJVwkJ' }, 'name': { 0: 'Måneskin', 1: 'The Kid LAROI', 2: 'Justin Bieber', 3: 'Lil Nas X', 4: 'Jack Harlow' } }, { 'job_id': { 0: 'test_job', 1: 'test_job', 2: 'test_job', 3: 'test_job', 4: 'test_job' }, 'spotify_track_id': { 0: '3Wrjm47oTz2sjIgck11l5e', 1: '5PjdY0CKGZdEuoNab3yDmX', 2: '5PjdY0CKGZdEuoNab3yDmX', 3: '27NovPIUIRrOZoCHxABJwK', 4: '27NovPIUIRrOZoCHxABJwK' }, 'spotify_artist_id': { 0: '0lAWpj5szCSwM4rUMHYmrr', 1: '2tIP7SsRs7vjIcLrU85W8J', 2: '1uNFoZAHBGtllmzznpCI3s', 3: '7jVv8c5Fj3E9VhNjxT4snq', 4: '2LIk90788K0zvyj2JJVwkJ' }, 'spotify_country_id': { 0: 'us', 1: 'us', 2: 'us', 3: 'us', 4: 'us' }, 'rank': { 0: 0, 1: 0, 2: 1, 3: 0, 4: 1 } } ), ] ) def test__extend_data_frame_with_public_data( s3_client: S3Client, main_service_no_db: Service, dim_track_data, dim_artist_data, link_track_artist_data ): market = 'us' df_json = get_df_source_data(market) data_frame = pd.DataFrame(df_json) main_service_no_db._vendor_api._send_request = mock.Mock(return_value=get_public_data(market)) enriched_data = main_service_no_db._extend_data_frame_with_public_data(data_frame) assert len(enriched_data) == 4 dim_track_dict = enriched_data[0].to_dict() dim_artist_dict = enriched_data[1].to_dict() link_track_artist_dict = enriched_data[2].to_dict() assert dim_track_data == dim_track_dict assert dim_artist_data == dim_artist_dict assert link_track_artist_data == link_track_artist_dict def test__store_enriched_dataframes( s3_client, s3_service: S3Service, main_service_no_db: Service, config_test ): market = 'global' df_json = get_df_source_data(market) data_frame = pd.DataFrame(df_json) main_service_no_db._vendor_api._send_request = mock.Mock(return_value=get_public_data(market)) enriched_data = main_service_no_db._extend_data_frame_with_public_data(data_frame) main_service_no_db._store_enriched_dataframes( enriched_data[0], enriched_data[1], enriched_data[2], enriched_data[3] ) bucket_files = s3_client.list_objects(Bucket=config_test.target_bucket) assert len(bucket_files['Contents']) == 5 metadata_df = wr.s3.read_parquet( f'{s3_service._build_target_directory_path()}/fact_spotify_metadata.parquet' ) result_df_data = metadata_df.to_dict() assert result_df_data