"""Unit tests for tasks of YouTube Asset Report Workflow.""" from datetime import datetime import re from unittest.mock import MagicMock from unittest.mock import patch from freezegun import freeze_time import pytest from feed_ingestion.flows.youtube_asset import config from feed_ingestion.flows.youtube_asset import tasks _date = '2018-11-16' class TestBootstrap(object): """Test bootstrap.""" @pytest.fixture(params=[ 'theorchard', 'sme', ]) def mock_s3(self, request): """Yield s3_mock, licensor.""" licensor = request.param s3utils_path = ( 'feed_ingestion.flows.youtube_asset.tasks.s3') files = [] if licensor == 'theorchard': files = [ 'content_owner_asset_a3.IODA.csv.gz', 'content_owner_asset_a3.ENT.csv.gz', # all below should be ignored 'asset_full_report_IODA_L_v1-2.csv.gz', 'asset_full_report_dmgi_L_v1-2.csv.gz', 'asset_full_report_dmgi_L_v1-0.csv.gz', 'asset_full_report_dmgi_L_v1-2.csv', 'ignore_me_full_report_dmgi_L_v1-2.csv.gz', ] if licensor == 'sme': files = [ '20201022_172646_00050_y255y_038fffee-d7a3-4197-b75b-a6487479db64', # noqa:E501 '20201022_172646_00050_y255y_05506ca6-f9fd-4968-b321-8fbfc298c639', # noqa:E501 # all below should be ignored 'NO20201022_172646_00050_y255y_05506ca6-f9fd-4968-b321-8fbfc298c639', # noqa:E501 'wrong_path/20201022_172646_00050_y255y_05506ca6-f9fd-4968-b321-8fbfc298c639', # noqa:E501 ] with patch(s3utils_path) as s3: s3.get_list_of_files_and_directories = MagicMock( return_value=files) s3.get_key_size = MagicMock(return_value=42) yield s3, licensor @pytest.fixture def expected_bootstrap_response(self): """Response for bootstrap task.""" date = datetime.strptime(_date, '%Y-%m-%d') return { 'date': _date, 'feed_name': 'youtube_asset_theorchard', 'skip_grab_reports_files': None, 'licensor': 'theorchard', 'credentials_path': config.credentials_paths.get('theorchard'), 'cms_dict': config.cms_dict, 's3_bucket': config.s3_bucket, 's3_dir_path': 's3://{}/{}'.format( config.s3_bucket, config.s3['archive_path'].format( date=date, licensor='theorchard') ), 's3_archive_path': config.s3['archive_path'].format( date=date, licensor='theorchard')} @freeze_time(_date) @patch('feed_ingestion.flows.youtube_asset.tasks.garcon_feed_status') def test_bootstrap_no_date_is_passed( self, feed_status_mock, expected_bootstrap_response): """Date should be set to today if it is not provided.""" context = { 'activity': MagicMock(), 'date': None, 'skip_grab_reports_files': None, 'licensor': 'theorchard', 'reload': False} result = tasks.bootstrap(**context) assert result == expected_bootstrap_response @patch('feed_ingestion.flows.youtube_asset.tasks.garcon_feed_status') def test_bootstrap_date_is_passed( self, feed_status_mock, expected_bootstrap_response): """Date should be returned if it is provided.""" context = { 'activity': MagicMock(), 'date': _date, 'skip_grab_reports_files': None, 'licensor': 'theorchard', 'reload': False} result = tasks.bootstrap(**context) assert result == expected_bootstrap_response @patch('feed_ingestion.flows.youtube_asset.tasks.garcon_feed_status') def test_source_files( self, feed_status_mock, mock_s3, expected_bootstrap_response): """Date should be returned if it is provided.""" s3, licensor = mock_s3 files = [] if licensor == 'theorchard': files = [{'file_name': 'content_owner_asset_a3.IODA.csv.gz', 'file_size': 42}, {'file_name': 'content_owner_asset_a3.ENT.csv.gz', 'file_size': 42}] if licensor == 'sme': files = [{ 'file_name': '20201022_172646_00050_y255y_038fffee-d7a3-4197-b75b-a6487479db64', # noqa: E501 'file_size': 42}, {'file_name': '20201022_172646_00050_y255y_05506ca6-f9fd-4968-b321-8fbfc298c639', # noqa: E501 'file_size': 42}] expected_result = {'source_files_dict': {'files': files}} result = tasks.source_files( activity=MagicMock(), licensor=licensor, s3_bucket=MagicMock(), s3_archive_path='/archive') assert result == expected_result @pytest.mark.parametrize('string, prefix, result', ( ('qwerty', 'qw', 'erty'), ('qwerty', 'qwerty', ''), ('qwerty', 'NOqwerty', 'qwerty'), ('/path/file', '/path/', 'file'), ('/path/file', '/path', '/file'), ('/path/file', '/nopath', '/path/file'), )) def test_remove_prefix(self, string, prefix, result): """Test remove_prefix function.""" assert tasks.remove_prefix(string, prefix) == result def test_sme_source_file_pattern(self): """Test generated parquet filenames created with Athena CTAS query.""" # sample filenames created by Athena Query filenames = [ '20201022_172646_00050_y255y_038fffee-d7a3-4197-b75b-a6487479db64', '20201022_172646_00050_y255y_05506ca6-f9fd-4968-b321-8fbfc298c636', '20201022_172646_00050_y255y_1d980cac-287b-424c-b2cf-06abd3f84aae', '20201022_172646_00050_y255y_24840d90-f760-4f5e-838b-f7f59c60245c', '20201022_172646_00050_y255y_37588958-24e8-4692-88f7-16cc812c5760', '20201022_172646_00050_y255y_38783bac-caa6-47dc-8c63-bb464465f428', '20201022_172646_00050_y255y_38fac22d-5519-4a76-886c-1cb51c9cbe87', '20201022_172646_00050_y255y_427b53f7-3537-43dd-a3ee-e09ac66d04bf', '20201022_172646_00050_y255y_4b6c601d-d642-4e6e-8851-2a2a118dcf23', '20201022_172646_00050_y255y_6bd57cdc-2518-47b7-8712-1ff722353261', ] file_pattern = config.source_file_pattern['sme'] for filename in filenames: assert re.match(file_pattern, filename), \ f"File {filename} doesn't match to sme_source_file_pattern"