"""Tests for the YouTube Asset Conflict flow.""" from unittest import mock from unittest.mock import MagicMock from garcon_contrib.aws.utils import garcon_s3 from feed_ingestion.flows.youtube_asset_conflict import config from feed_ingestion.flows.youtube_asset_conflict.flow import Flow context_date = '2017-01-01' date_yyyymmdd = context_date.replace('-', '') file_names = [ 'asset_conflict_report_theorchardmusic_N_v1-0.csv', 'asset_conflict_report_IODA_N_v1-0.csv' ] drop_bucket = 'drop_bucket' archive_bucket = 'archive_bucket' staging_raw_temp_tables = [ config.snowflake_table_names.get( 'staging_raw_temp').format( account=account.lower(), datestamp=date_yyyymmdd) for account in config.accounts ] staging_raw_temp_table_relationships = [] for account in config.accounts: file_name = config.source_filename_template.format( account=account ) temp_table_name = config.snowflake_table_names['staging_raw_temp'].format( account=account.lower(), datestamp=date_yyyymmdd) staging_raw_temp_table_relationships.append( (temp_table_name, file_name, account.lower())) context = { 'bootstrap.drop_file_names': file_names, 'bootstrap.s3_archive_path': config.s3.get('archive').format( s3_bucket=archive_bucket, s3_feed_name=config.s3_feed_name, datestamp=context_date), 'bootstrap.s3_staging_raw_temp': config.s3.get( 'staging_raw_temp').format( s3_bucket=archive_bucket, s3_feed_name=config.s3_feed_name, datestamp=context_date), 'bootstrap.staging_raw_temp_table_relationships': staging_raw_temp_table_relationships } def test_youtube_asset_conflict_sf_flow_decider(): """Test normal decider execution.""" youtube_asset_conflict_flow = Flow() schedule = MagicMock() schedule_result_object = MagicMock() schedule_result_object.result = {} schedule.return_value = schedule_result_object youtube_asset_conflict_flow.decider( schedule, {'context_date': context_date}) schedule.assert_has_calls([ mock.call('bootstrap', mock.ANY), mock.call('grab_reports_files', mock.ANY, requires=[mock.ANY]), mock.call('source_files', mock.ANY, requires=[mock.ANY]), mock.call( 'move_files_to_staging_raw_temp_bucket', mock.ANY, requires=[mock.ANY]), mock.call( 'create_temp_staging_tables', mock.ANY, requires=[mock.ANY]), mock.call( 'load_temp_staging_table', mock.ANY, requires=[mock.ANY] ), mock.call( 'truncate_staging_raw_table', mock.ANY, requires=[mock.ANY] ), mock.call( 'insert_into_staging_raw_table', mock.ANY, requires=[mock.ANY] ), mock.call( 'create_fact_conflict_temp_table', mock.ANY, requires=[mock.ANY] ), mock.call( 'drop_create_territories_temp_table', mock.ANY, requires=[mock.ANY] ), mock.call( 'fill_territories_temp_table', mock.ANY, requires=[mock.ANY] ), mock.call( 'create_youtube_asset_conflict_by_territory_table', mock.ANY, requires=[mock.ANY] ), mock.call( 'fill_youtube_asset_conflict_by_territory_table', mock.ANY, requires=[mock.ANY] ), mock.call( 'create_fact_conflict_temp_table', mock.ANY, requires=[mock.ANY] ), mock.call( 'insert_overwrite_into_fact_conflict_temp_table', mock.ANY, requires=[mock.ANY] ), mock.call( 'check_size_of_fact_conflict_temp_table', mock.ANY, requires=[mock.ANY] ), mock.call( 'check_number_of_unresolved_conflicts', mock.ANY, requires=[mock.ANY] ), mock.call( 'insert_into_fact_conflict_table', mock.ANY, requires=[mock.ANY] ), mock.call( 'update_fact_conflict_resolved_datetime', mock.ANY, requires=[mock.ANY] ), mock.call( 'reset_es_indexed_for_partially_resolved_conflicts', mock.ANY, requires=[mock.ANY] ), mock.call( 'update_fact_conflict_yt_recent_daily_average', mock.ANY, requires=[mock.ANY] ), mock.call( 'update_fact_conflict_views_in_conflict', mock.ANY, requires=[mock.ANY] ), mock.call( 'drop_staging_raw_temp_tables', mock.ANY, requires=[mock.ANY] ), mock.call( 'drop_yact_table', mock.ANY, requires=[mock.ANY] ), mock.call( 'drop_fact_conflict_temp_table', mock.ANY, requires=[mock.ANY] ), mock.call( 'set_overall_status_ingested', mock.ANY, requires=[mock.ANY] ) ]) def test_staging_raw_temp_files_generator(): """Test staging_raw_temp_files_generator method.""" youtube_asset_conflict_flow = Flow() # generated s3 file locations generator = list( youtube_asset_conflict_flow.staging_raw_temp_files_generator(context)) for file_name in file_names: file_info = { 'source_key_name': garcon_s3.extract_bucket_path(context.get( 'bootstrap.s3_archive_path'))[1] + file_name, 'destination_key_name': garcon_s3.extract_bucket_path(context.get( 'bootstrap.s3_staging_raw_temp'))[1] + file_name} assert file_info in generator def test_staging_raw_temp_tables_relationships_generator(): """Test staging_raw_temp_tables_relationships_generator.""" flow = Flow() # generated temp staging raw table relationships generator = list( flow.staging_raw_temp_tables_relationships_generator(context)) for table_name, drop_file, co in staging_raw_temp_table_relationships: temp_table_relationship = { 'staging_raw_temp_table': table_name, 'drop_file': drop_file, 'content_owner': co.lower()} assert temp_table_relationship in generator