# pylint: disable=unused-argument import datetime import pathlib import pytest from db_schema.factories import apps from apps_etl_manager.entities.config import UnitOfWorkConfig, UnitOfWorkConfigs, from_file from apps_etl_manager.entities.entities import EtlArgs def test_from_file(): path = pathlib.Path(__file__).parent / 'fixtures' / 'test_etl_config.json' assert len(from_file(str(path))) == 1 @pytest.mark.integration def test_unit_of_work_list(session): apps_uow = apps.UnitOfWorkFactory( unit_of_work_code='apple-20200202-streams', report_date=datetime.date(2020, 2, 2) ) config = UnitOfWorkConfig( unit_of_work='apple-{yyyymmdd}-streams', schedule='*/10 * * * *', data_source=['slz'], args=EtlArgs( dsp='apple', unit_type='daily', report_type='streams', dbx_job_id=1, days_offset=1, valid_from='2019-01-01', dbx_job_max_lifetime=1800, ), dependencies=[], ) unit_of_works = UnitOfWorkConfigs([config]) assert unit_of_works.get(apps_uow) == config @pytest.mark.integration def test_unit_of_work_list_append(session): apps_uow = apps.UnitOfWorkFactory( unit_of_work_code='apple-20200202-streams', report_date=datetime.date(2020, 2, 2) ) config = UnitOfWorkConfig( unit_of_work='apple-{yyyymmdd}-streams', schedule='*/10 * * * *', data_source=['slz'], args=EtlArgs( dsp='apple', unit_type='daily', report_type='streams', dbx_job_id=1, days_offset=1, valid_from='2019-01-01', dbx_job_max_lifetime=1800, ), dependencies=[], ) unit_of_works = UnitOfWorkConfigs() unit_of_works.append(config) assert unit_of_works.get(apps_uow) == config @pytest.mark.integration def test_unit_of_work_list_get_return_none_no_config(session): apps_uow = apps.UnitOfWorkFactory( unit_of_work_code='apple-20210101-streams', report_date=datetime.date(2021, 1, 1) ) unit_of_works = UnitOfWorkConfigs([]) assert unit_of_works.get(apps_uow) is None @pytest.mark.integration def test_unit_of_work_list_get_return_none_for_not_valid(session): apps_uow = apps.UnitOfWorkFactory( unit_of_work_code='apple-20210101-streams', report_date=datetime.date(2021, 1, 1) ) config = UnitOfWorkConfig( unit_of_work='apple-{yyyymmdd}-streams', schedule='*/10 * * * *', data_source=['slz'], args=EtlArgs( dsp='apple', unit_type='daily', report_type='streams', dbx_job_id=1, days_offset=1, valid_from='2019-01-01', valid_until='2020-01-01', dbx_job_max_lifetime=1800, ), dependencies=[], ) unit_of_works = UnitOfWorkConfigs([config]) assert unit_of_works.get(apps_uow) is None @pytest.mark.parametrize( 'lambda_date,frequency,days_offset,is_valid', ( (datetime.date(2020, 3, 3), '0 0 * * 7', 2, True), # Sunday report on Tuesday (datetime.date(2020, 3, 2), '0 0 * * 7', 1, True), # Sunday report on Monday (datetime.date(2020, 3, 2), '0 0 * * 7', 2, False), # Sunday report on Monday (datetime.date(2020, 2, 2), '0 0 1 * *', 1, True), # on day-of-month 1 (datetime.date(2020, 2, 2), '0 0 2 * *', 1, False), # on day-of-month 2 ) ) def test_is_valid_with_frequency_and_days_offset(lambda_date, frequency, days_offset, is_valid): config = UnitOfWorkConfig( unit_of_work='apple-{yyyymmdd}-streams', schedule='*/10 * * * *', data_source='slz', frequency=frequency, args=EtlArgs( dsp='apple', unit_type='daily', report_type='streams', dbx_job_id=1, days_offset=days_offset, valid_from='2019-01-01', dbx_job_max_lifetime=1800, ), dependencies=[], ) assert config.is_valid_to_initialize(lambda_date, ['slz']) is is_valid @pytest.mark.parametrize( 'uow_data_source,payload_data_sources,is_valid', ( ('slz', ['slz', 'chartmetric'], True), ('slz', ['slz'], True), ('slz', ['chartmetric'], False), ('slz', [], False), ('slz', ['another_source'], False), ) ) def test_is_valid_by_data_source(uow_data_source, payload_data_sources, is_valid): config = UnitOfWorkConfig( unit_of_work='apple-{yyyymmdd}-streams', schedule='*/10 * * * *', data_source=uow_data_source, args=EtlArgs( dsp='apple', unit_type='daily', report_type='streams', dbx_job_id=1, days_offset=1, valid_from='2019-01-01', dbx_job_max_lifetime=1800, ), dependencies=[], ) assert config.is_valid_to_initialize(datetime.date.today(), payload_data_sources) is is_valid @pytest.mark.parametrize( 'valid_from,valid_until,lambda_date,is_valid', ( ('2017-06-30', '2021-04-13', datetime.date(2021, 4, 12), True), ('2017-06-30', '2021-04-13', datetime.date(2021, 4, 13), True), ('2017-06-30', '2021-04-13', datetime.date(2021, 4, 14), True), ('2017-06-30', '2021-04-13', datetime.date(2021, 4, 15), False), ('2021-04-14', None, datetime.date(2021, 4, 14), False), ('2021-04-14', None, datetime.date(2021, 4, 15), True), ('2021-04-14', None, datetime.date(2021, 4, 16), True), ) ) def test_is_valid_to_initialize_by_valid_dates(valid_from, valid_until, lambda_date, is_valid): config = UnitOfWorkConfig( unit_of_work='apple-{yyyymmdd}-streams_artist_date_day', schedule='*/10 * * * *', data_source='slz', args=EtlArgs( dsp='apple', unit_type='daily', report_type='streams_artist_date_day', dbx_job_id=1, days_offset=1, valid_from=valid_from, valid_until=valid_until, dbx_job_max_lifetime=1800, ), dependencies=[], ) assert config.is_valid_to_initialize(lambda_date, 'slz') is is_valid @pytest.mark.parametrize( 'valid_from,valid_until,report_date,is_valid', ( ('2017-06-30', '2021-04-13', datetime.date(2021, 4, 12), True), ('2017-06-30', '2021-04-13', datetime.date(2021, 4, 13), True), ('2017-06-30', '2021-04-13', datetime.date(2021, 4, 14), False), ('2017-06-30', '2021-04-13', datetime.date(2021, 4, 15), False), ('2021-04-14', None, datetime.date(2021, 4, 13), False), ('2021-04-14', None, datetime.date(2021, 4, 14), True), ('2021-04-14', None, datetime.date(2021, 4, 15), True), ) ) def test_is_valid_to_run_by_valid_dates(valid_from, valid_until, report_date, is_valid): config = UnitOfWorkConfig( unit_of_work='apple-{yyyymmdd}-streams_artist_date_day', schedule='*/10 * * * *', data_source='slz', args=EtlArgs( dsp='apple', unit_type='daily', report_type='streams_artist_date_day', dbx_job_id=1, days_offset=1, valid_from=valid_from, valid_until=valid_until, dbx_job_max_lifetime=1800, ), dependencies=[], ) assert config.is_valid_to_run(report_date) is is_valid @pytest.mark.parametrize( 'report_date,configs_count', ( (datetime.date(2014, 3, 14), 0), (datetime.date(2017, 7, 14), 1), (datetime.date(2017, 9, 14), 2), (datetime.date(2021, 4, 14), 2), ) ) def test_is_valid_to_run_by_report_date(report_date, configs_count): path = pathlib.Path(__file__).parent / 'fixtures' / 'test_etl_config_double_uows.json' uow_configs = from_file(str(path)) valid_uow_configs = [conf for conf in uow_configs if conf.is_valid_to_run(report_date)] assert len(uow_configs) == 4 assert len(valid_uow_configs) == configs_count