import os import boto3 import pytest from garcon_contrib.dynamo_feed_status import garcon_feed_status EXISTING_ITEM = { 'date': {'S': '2024-08-01'}, 'feed_name': {'S': 'test_feed_name'}, 'filename_one_status': {'S': 'NOT_AVAILABLE'}, 'status': {'S': 'INGESTED'} } STATUSES = [ garcon_feed_status.STATUS_INGESTED, garcon_feed_status.STATUS_NOT_INGESTED, garcon_feed_status.STATUS_NOT_AVAILABLE, garcon_feed_status.STATUS_DOWNLOADED, garcon_feed_status.STATUS_POPULATED_RAW_TABLE, garcon_feed_status.STATUS_INGESTED_TO_MYSQL, ] DYNAMODB_TABLE = os.environ['FEED_INGESTION_TABLE'] date = '2024-08-01' feed_name = 'test_feed_name' def _delete_item(feed_name, date): boto3.client('dynamodb').delete_item( TableName=DYNAMODB_TABLE, Key={ 'feed_name': {'S': feed_name}, 'date': {'S': date} } ) def _get_item(feed_name, date): return boto3.client('dynamodb').get_item( TableName=DYNAMODB_TABLE, Key={ 'feed_name': {'S': feed_name}, 'date': {'S': date} } ).get('Item') def _put_item(feed_name, date, item): return boto3.client('dynamodb').put_item( TableName=DYNAMODB_TABLE, Item=item, ) def _get_args_kwargs(kwargs, mode): if mode == 'args': args = kwargs.values() kwargs = {} elif mode == 'kwargs': args = () else: raise ValueError return args, kwargs @pytest.fixture() def existing_item(): item = EXISTING_ITEM _put_item(feed_name, date, item) yield item _delete_item(feed_name, date) class TestGarconFeedStatus: @pytest.mark.parametrize('mode', ['args', 'kwargs']) def test_set_missing_files(self, mode): _delete_item(feed_name, date) kwargs = dict(feed_name=feed_name, date=date, missing_files=['missing_file1', 'missing_file2']) args, kwargs = _get_args_kwargs(kwargs, mode) result = garcon_feed_status.set_missing_files( *args, **kwargs) assert result == None item = _get_item(feed_name, date) assert item == { 'date': {'S': '2024-08-01'}, 'feed_name': {'S': 'test_feed_name'}, 'missing_files': {'S': 'missing_file1,missing_file2'}} @pytest.mark.parametrize('mode', ['args', 'kwargs']) @pytest.mark.parametrize('status', STATUSES[0:1]) def test_set_status(self, mode, status): _delete_item(feed_name, date) kwargs = dict( feed_name=feed_name, date=date, file_name='filename_one.csv', status=status) args, kwargs = _get_args_kwargs(kwargs, mode) result = garcon_feed_status.set_status( *args, **kwargs) assert result == None item = _get_item(feed_name, date) assert item == { 'date': {'S': '2024-08-01'}, 'feed_name': {'S': 'test_feed_name'}, 'filename_one_status': {'S': status}} @pytest.mark.parametrize('mode', ['args', 'kwargs']) def test_set_overall_status(self, mode): _delete_item(feed_name, date) status = garcon_feed_status.STATUS_NOT_INGESTED kwargs = dict(feed_name=feed_name, date=date, overall_status=status) args, kwargs = _get_args_kwargs(kwargs, mode) result = garcon_feed_status.set_overall_status( *args, **kwargs) assert result == None item = _get_item(feed_name, date) assert item == { 'date': {'S': date}, 'feed_name': {'S': feed_name}, 'status': {'S': status}} def test_set_overall_status_set_status(self): _delete_item(feed_name, date) garcon_feed_status.set_overall_status( feed_name=feed_name, date=date, overall_status=garcon_feed_status.STATUS_NOT_INGESTED ) garcon_feed_status.set_status( feed_name=feed_name, date=date, file_name='filename_one.csv', status=garcon_feed_status.STATUS_NOT_AVAILABLE ) item = _get_item(feed_name, date) assert item == { 'date': {'S': '2024-08-01'}, 'feed_name': {'S': 'test_feed_name'}, 'filename_one_status': {'S': 'NOT_AVAILABLE'}, 'status': {'S': 'NOT_INGESTED'} } @pytest.mark.parametrize('mode', ['args', 'kwargs']) def test_get_overall_status_existing(self, mode, existing_item): kwargs = dict(feed_name=feed_name, date=date) args, kwargs = _get_args_kwargs(kwargs, mode) result = garcon_feed_status.get_overall_status( *args, **kwargs) assert result == 'INGESTED' assert _get_item(feed_name, date) == existing_item def test_get_overall_status_not_existing(self): kwargs = dict(feed_name=feed_name, date=date) args, kwargs = _get_args_kwargs(kwargs, 'kwargs') result = garcon_feed_status.get_overall_status( *args, **kwargs) assert result == False item = _get_item(feed_name, date) assert item == None @pytest.mark.parametrize('mode', ['kwargs']) def test_get_status_existing(self, mode, existing_item): kwargs = dict( feed_name=feed_name, date=date, file_name='filename_one.csv') args, kwargs = _get_args_kwargs(kwargs, mode) result = garcon_feed_status.get_status( *args, **kwargs) assert result == 'NOT_AVAILABLE' assert _get_item(feed_name, date) == existing_item @pytest.mark.parametrize('mode', ['kwargs']) def test_get_status_not_existing(self, mode): kwargs = dict( feed_name=feed_name, date=date, file_name='filename_one.csv') args, kwargs = _get_args_kwargs(kwargs, mode) result = garcon_feed_status.get_status( *args, **kwargs) assert result == None assert _get_item(feed_name, date) == None @pytest.mark.parametrize('mode', ['kwargs']) def test_delete_status_existing(self, mode, existing_item): kwargs = dict( feed_name=feed_name, date=date) args, kwargs = _get_args_kwargs(kwargs, mode) result = garcon_feed_status.delete_status( *args, **kwargs) assert result == None assert _get_item(feed_name, date) == None @pytest.mark.parametrize('mode', ['kwargs']) def test_delete_status_not_existing(self, mode): kwargs = dict( feed_name=feed_name, date=date) args, kwargs = _get_args_kwargs(kwargs, mode) result = garcon_feed_status.delete_status( *args, **kwargs) assert result == None assert _get_item(feed_name, date) == None def test_get_missing_dates(self, existing_item): missing_dates = garcon_feed_status.get_missing_dates(feed_name) assert len(missing_dates) > 10 def test_find_non_processed_date(self, existing_item): existing_date = existing_item['date']['S'] available_dates = [existing_date, '2022-01-02'] non_processed_date = garcon_feed_status.find_non_processed_date(feed_name, available_dates) assert non_processed_date == '2022-01-02'