# pylint: disable=protected-access, unused-argument import os import pytest import smart_open from db_schema.schemas import slz from lxml.etree import parse from slz_ddex_downloader import exceptions from slz_ddex_downloader.entities import AcknowledgementFileStatusEnum from .utils import create_release @pytest.mark.integration def test__move_release_to_archive_bucket(release_service, s3_service, config_test): release = release_service._release release_dir = os.path.join('/tmp', release.folder_name) expected = [ 'ddex/release_message/v1/report_date=2021-04-27/report_licensor=sme/valid_release.tar.gz' ] release_service._move_release_to_archive_bucket() s3_path = os.path.join(release.upload_prefix, release.folder_name) keys = s3_service.get_keys_by_folder(config_test.archive_bucket, s3_path) assert keys == expected assert not list(os.walk(release_dir)) @pytest.mark.integration def test__upload_acknowledgement_file( acknowledgement_test, release_service, s3_service, config_test ): release_service._upload_acknowledgement_file(acknowledgement_test) s3_path = os.path.join('acknowledgements', acknowledgement_test.file_name) keys = s3_service.get_keys_by_folder(config_test.feed_bucket, 'acknowledgements') assert keys == [s3_path] @pytest.mark.integration def test__create_release_obj_filter_invalid_formats(release_service, s3_service, config_test): invalid_keys = [ 'valid_release/resources/archive.gz', 'valid_release/resources/video.mp4', 'valid_release/resources/img.svg', ] valid_keys = [ 'valid_release/resources/img.jpeg', 'valid_release/resources/img.jpg', 'valid_release/resources/img.png', 'valid_release/resources/img.gif', 'valid_release/resources/img.tiff', 'valid_release/resources/img.raw', 'valid_release/resources/img.bmp', ] content = 'content' for key in invalid_keys + valid_keys: s3_service._s3_client.put_object(Bucket=config_test.feed_bucket, Key=key, Body=content) release = release_service._create_release_obj() release_resources = [file.key for file in release.files] # check that invalid resources formats have been filtered for key in invalid_keys: assert key not in release_resources for key in valid_keys: assert key in release_resources @pytest.mark.integration def test_process_ok(release_service, unit_of_work, db, s3_service, config_test, clean_db): release_service.process() release_title = release_service._release_title # check content_status content_status = db.session.query(slz.ContentStatus).filter_by(context=release_title).one() assert content_status.content_status == slz.ContentStatusEnum.COMPLETE # check quarantine and corrupted buckets are empty assert not s3_service.get_keys_by_folder( config_test.decompressed_quarantine_bucket, release_title ) assert not s3_service.get_keys_by_folder(config_test.corrupted_bucket, release_title) # check decompressed and archive buckets contains release prefix = release_service._payload.upload_prefix release_files = [ f'{prefix}/{release_title}/A10301A0001357189C.xml', f'{prefix}/{release_title}/resources/A10301A0001357189C_T-102043747899_Image.jpg' ] keys = s3_service.get_keys_by_folder(config_test.decompressed_bucket, prefix) assert keys == release_files keys = s3_service.get_keys_by_folder(config_test.archive_bucket, prefix) assert keys == [f'{release_service._payload.upload_prefix}/{release_title}.tar.gz'] # check acknowledgement file presence akn_file_path = s3_service.get_keys_by_folder( config_test.feed_bucket, config_test.acknowledgements_folder )[0] assert akn_file_path # check acknowledgement file_status with smart_open.open(f's3://{config_test.feed_bucket}/{akn_file_path}', 'r') as fin: akn_file = parse(fin) assert akn_file.xpath('FileStatus')[0].text == 'FileOK' @pytest.mark.parametrize( 'release_title, file_status', [ ('unvalid_release_by_resources', AcknowledgementFileStatusEnum.RESOURCE_MISSING.value), ('unvalid_release_by_xml_content', AcknowledgementFileStatusEnum.INTERNAL_ERROR.value), ( 'unvalid_release_by_wrong_extension', AcknowledgementFileStatusEnum.NEW_RELEASE_MESSAGE_INVALID.value ), ( 'unvalid_release_by_unparseable_xml', AcknowledgementFileStatusEnum.NEW_RELEASE_MESSAGE_INVALID.value ), ] ) @pytest.mark.integration def test_process_failed( release_service, db, unit_of_work, s3_service, config_test, release_title, file_status, clean_db ): content_status = slz.ContentStatus( unit_of_work_id=unit_of_work.unit_of_work_id, context=release_title, failure_count=2, latest_job_id='ddex-20210427-sme-release_message-v1_26640_20210428T12.14.07', ) release = create_release(release_service, release_title) release_service._release = release with pytest.raises(exceptions.XMLValidationError): release_service.process() # check content_status content_status = db.session.query(slz.ContentStatus).filter_by(context=release_title).one() assert content_status.content_status == slz.ContentStatusEnum.FAILED assert content_status.failure_count == 3 # check content_failure_log assert content_status.content_failure_logs.first() # check quarantine bucket is empty assert not s3_service.get_keys_by_folder( config_test.decompressed_quarantine_bucket, release_title ) # check corrupted bucket contains release assert s3_service.get_keys_by_folder(config_test.corrupted_bucket, release_title) # check decompressed and archive buckets are empty prefix = release_service._payload.upload_prefix assert not s3_service.get_keys_by_folder(config_test.decompressed_bucket, prefix) assert not s3_service.get_keys_by_folder(config_test.archive_bucket, prefix) # check acknowledgement file presence akn_file_path = s3_service.get_keys_by_folder( config_test.feed_bucket, config_test.acknowledgements_folder )[0] assert akn_file_path # check acknowledgement file_status with smart_open.open(f's3://{config_test.feed_bucket}/{akn_file_path}', 'r') as fin: akn_file = parse(fin) assert akn_file.xpath('FileStatus')[0].text == file_status