"""spotify_artificial_streams flow bootstrap task tests.""" __all__ = [ 'TestBootstrapTask', ] from datetime import datetime from unittest.mock import patch from dateutil.relativedelta import relativedelta from feed_ingestion.flows.spotify_artificial_streams import config from feed_ingestion.flows.spotify_artificial_streams.tasks import bootstrap from tests.flows.spotify_artificial_streams.constants import garcon_methods from tests.flows.spotify_artificial_streams.tasks import base_test_case class TestBootstrapTask(base_test_case.BaseSpotifyArtificialStreamsTestTask): """Test cases for spotify_artificial_streams flow bootstrap task.""" @patch(garcon_methods['delete_status']) def test_bootstrap_reload_date_present(self, delete_status): """Test bootstrap task on reload and the load date in context.""" bootstrap_kwargs = { 'activity': self.activity, 'reload': 'True', 'date': self.string_date, } # prepared calculations current_datetime = datetime.fromisoformat(self.string_date) ( blob_file_name_wildcard, drop_blob_bucket, drop_blob_path, full_drop_blob_names, ) = self._prepare_blob_wildcard_and_full_name( current_datetime=current_datetime, ) temp_staging_raw_name = ( 'TEMP_STAGING_RAW_SPOTIFY_ARTIFICIAL_STREAMS' '_{current_datetime:%Y%m%d}' ).format( current_datetime=current_datetime, ) archive_blob_prefix = '{archive_blob_prefix}/{date}'.format( archive_blob_prefix=config.s3_blobs_prefixes[ 'archive_blob_prefix' ], date=current_datetime.date().isoformat(), ) response = bootstrap(**bootstrap_kwargs) expected_response = { 'archive_blob_prefix': archive_blob_prefix, 'date': current_datetime.date().isoformat(), 'drop_bucket_name': drop_blob_bucket, 'drop_blob_path': drop_blob_path, 'drop_blob_wildcard': blob_file_name_wildcard, 'feed_name': 'spotify_artificial_streams', 'secrets_path': config.secrets_path, 'staging_raw_load_kwargs': { 'blob_path': full_drop_blob_names, }, 'temp_staging_raw_load_kwargs': { 'file_pattern': blob_file_name_wildcard, 'snowflake_error_limit': config.snowflake_error_limit, }, 'temp_staging_raw_name': temp_staging_raw_name, 'snowflake_error_limit': config.snowflake_error_limit, 'staging_raw_table': config.staging_raw_table_name, } self.assertDictEqual(response, expected_response) @patch(garcon_methods['delete_status']) def test_bootstrap_reload_no_date(self, delete_status): """Test bootstrap task on reload and no load date in context.""" bootstrap_kwargs = { 'activity': self.activity, 'reload': 'True', 'date': self.string_date, } # prepared calculations current_datetime = datetime.now() ( blob_file_name_wildcard, drop_blob_bucket, drop_blob_path, full_drop_blob_names, ) = self._prepare_blob_wildcard_and_full_name( current_datetime=current_datetime, ) temp_staging_raw_name = ( 'TEMP_STAGING_RAW_SPOTIFY_ARTIFICIAL_STREAMS' '_{current_datetime:%Y%m%d}' ).format( current_datetime=current_datetime, ) archive_blob_prefix = '{archive_blob_prefix}/{date}'.format( archive_blob_prefix=config.s3_blobs_prefixes[ 'archive_blob_prefix' ], date=current_datetime.date().isoformat(), ) response = bootstrap(**bootstrap_kwargs) expected_response = { 'archive_blob_prefix': archive_blob_prefix, 'date': current_datetime.date().isoformat(), 'drop_bucket_name': drop_blob_bucket, 'drop_blob_path': drop_blob_path, 'drop_blob_wildcard': blob_file_name_wildcard, 'feed_name': 'spotify_artificial_streams', 'secrets_path': config.secrets_path, 'staging_raw_load_kwargs': { 'blob_path': full_drop_blob_names, }, 'temp_staging_raw_load_kwargs': { 'file_pattern': blob_file_name_wildcard, 'snowflake_error_limit': config.snowflake_error_limit, }, 'temp_staging_raw_name': temp_staging_raw_name, 'snowflake_error_limit': config.snowflake_error_limit, 'staging_raw_table': config.staging_raw_table_name, } self.assertDictEqual(response, expected_response) @patch(garcon_methods['delete_status']) def test_bootstrap_not_reload_date_present(self, delete_status): """Test bootstrap task not on reload and the load date in context.""" bootstrap_kwargs = { 'activity': self.activity, 'reload': 'False', 'date': self.string_date, } # prepared calculations current_datetime = datetime.fromisoformat(self.string_date) ( blob_file_name_wildcard, drop_blob_bucket, drop_blob_path, full_drop_blob_names, ) = self._prepare_blob_wildcard_and_full_name( current_datetime=current_datetime, ) temp_staging_raw_name = ( 'TEMP_STAGING_RAW_SPOTIFY_ARTIFICIAL_STREAMS' '_{current_datetime:%Y%m%d}' ).format( current_datetime=current_datetime, ) archive_blob_prefix = '{archive_blob_prefix}/{date}'.format( archive_blob_prefix=config.s3_blobs_prefixes[ 'archive_blob_prefix' ], date=current_datetime.date().isoformat(), ) response = bootstrap(**bootstrap_kwargs) expected_response = { 'archive_blob_prefix': archive_blob_prefix, 'date': current_datetime.date().isoformat(), 'drop_bucket_name': drop_blob_bucket, 'drop_blob_path': drop_blob_path, 'drop_blob_wildcard': blob_file_name_wildcard, 'feed_name': 'spotify_artificial_streams', 'secrets_path': config.secrets_path, 'staging_raw_load_kwargs': { 'blob_path': full_drop_blob_names, }, 'temp_staging_raw_load_kwargs': { 'file_pattern': blob_file_name_wildcard, 'snowflake_error_limit': config.snowflake_error_limit, }, 'temp_staging_raw_name': temp_staging_raw_name, 'snowflake_error_limit': config.snowflake_error_limit, 'staging_raw_table': config.staging_raw_table_name, } self.assertDictEqual(response, expected_response) @patch(garcon_methods['delete_status']) def test_bootstrap_no_reload_not_date_present(self, delete_status): """Test bootstrap task not on reload and no load date in context.""" bootstrap_kwargs = { 'activity': self.activity, 'reload': 'False', 'date': self.string_date, } # prepared calculations current_datetime = datetime.now() ( blob_file_name_wildcard, drop_blob_bucket, drop_blob_path, full_drop_blob_names, ) = self._prepare_blob_wildcard_and_full_name( current_datetime=current_datetime, ) temp_staging_raw_name = ( 'TEMP_STAGING_RAW_SPOTIFY_ARTIFICIAL_STREAMS' '_{current_datetime:%Y%m%d}' ).format( current_datetime=current_datetime, ) archive_blob_prefix = '{archive_blob_prefix}/{date}'.format( archive_blob_prefix=config.s3_blobs_prefixes[ 'archive_blob_prefix' ], date=current_datetime.date().isoformat(), ) response = bootstrap(**bootstrap_kwargs) expected_response = { 'archive_blob_prefix': archive_blob_prefix, 'date': current_datetime.date().isoformat(), 'drop_bucket_name': drop_blob_bucket, 'drop_blob_path': drop_blob_path, 'drop_blob_wildcard': blob_file_name_wildcard, 'feed_name': 'spotify_artificial_streams', 'secrets_path': config.secrets_path, 'staging_raw_load_kwargs': { 'blob_path': full_drop_blob_names, }, 'temp_staging_raw_load_kwargs': { 'file_pattern': blob_file_name_wildcard, 'snowflake_error_limit': config.snowflake_error_limit, }, 'temp_staging_raw_name': temp_staging_raw_name, 'snowflake_error_limit': config.snowflake_error_limit, 'staging_raw_table': config.staging_raw_table_name, } self.assertDictEqual(response, expected_response) def _prepare_blob_wildcard_and_full_name( self, current_datetime: datetime, ): previous_month_datetime = current_datetime - relativedelta(months=1) first_day_of_the_month_str = '{current_month_id}01'.format( current_month_id=previous_month_datetime.strftime('%Y%m'), ) last_day_of_the_month_str = ( previous_month_datetime + relativedelta(day=31) ).strftime('%Y%m%d') blob_file_name_wildcard = ( f'.*{config.blob_name_prefix}' f'-{first_day_of_the_month_str}' f'-{last_day_of_the_month_str}' f'.*.{config.blob_name_extension}' ) drop_blob_bucket = config.s3_blobs_prefixes['drop_blob_bucket'] drop_blob_path = config.s3_blobs_prefixes['drop_blob_path'] full_drop_blob_names = ( f's3://{drop_blob_bucket}/{drop_blob_path}' f'/{blob_file_name_wildcard}' ) return ( blob_file_name_wildcard, drop_blob_bucket, drop_blob_path, full_drop_blob_names, )