# pylint: disable=redefined-outer-name import os from datetime import datetime from time import sleep from unittest import mock import boto3 import pytest import smart_open from boto3_type_annotations.s3 import Client as S3Client from db_schema.postgres.connection import Connection from db_schema.schemas import slz from moto import mock_s3 from slz_ddex_downloader.entities import Acknowledgement, LambdaConfig, Payload from slz_ddex_downloader.services.db import DBService from slz_ddex_downloader.services.release import ReleaseService from slz_ddex_downloader.services.releases import ReleasesService from slz_ddex_downloader.services.s3 import S3Service from slz_ddex_downloader.services.validator import Validator from . import FIXTURES_PATH from .utils import upload_local_releases @pytest.fixture(scope='session') def db() -> Connection: def creds_loader(): return dict( active_endpoint=os.environ.get('PG_HOST', '0.0.0.0'), port=os.environ.get('PG_PORT', 5432), database=os.environ.get('PG_DB', 'slz'), username=os.environ.get('PG_USER', 'admin'), password=os.environ.get('PG_PASSWORD', 'admin'), ) conn = Connection(credentials_loader=creds_loader, name='housekeeping_test', version='v1') while True: result = conn.session.execute( # pylint: disable=no-member 'SELECT id FROM databasechangelog ORDER BY dateexecuted DESC LIMIT 1' ) if result.first(): break sleep(1) yield conn @pytest.fixture(scope='function') def clean_db(db): yield for model in [slz.ContentFailureLog, slz.ContentStatus, slz.UnitOfWork]: db.session.query(model).delete() db.session.commit() db.session.close() @pytest.fixture def config_test(): return LambdaConfig( environment='test', rds_secrets_key='rds_secrets_key', feed_bucket='feed_bucket', decompressed_quarantine_bucket='decompressed_quarantine_bucket', archive_bucket='archive_bucket', decompressed_bucket='decompressed_bucket', corrupted_bucket='corrupted_bucket', acknowledgements_folder='acknowledgements', release_batch_size=3, sentry_secret_key='sentry_secret_key' ) @pytest.fixture def payload_test(): return Payload( uow_code='ddex-20210427-sme-release_message-v1', dsp='ddex', report_type='release_message', version='v1', report_date='2021-04-27', licensor='sme', config_bucket='config_bucket', job_id='test_job' ) @pytest.fixture def acknowledgement_test(): return Acknowledgement(file_name='acknowledgement_file_name.xml', content='content', size=42) @pytest.fixture(scope='function', autouse=True) def aws_credentials(): os.environ['AWS_ACCESS_KEY_ID'] = 'testing' os.environ['AWS_SECRET_ACCESS_KEY'] = 'testing' os.environ['AWS_SECURITY_TOKEN'] = 'testing' os.environ['AWS_SESSION_TOKEN'] = 'testing' os.environ['AWS_DEFAULT_REGION'] = 'us-east-1' boto3.setup_default_session( aws_access_key_id='testing', aws_secret_access_key='testing', aws_session_token='testing', region_name='testing', ) @pytest.fixture def s3_client(payload_test, config_test): with mock_s3(): buckets = [ payload_test.config_bucket, config_test.feed_bucket, config_test.decompressed_quarantine_bucket, config_test.archive_bucket, config_test.decompressed_bucket, config_test.corrupted_bucket, ] s3_client_: S3Client = boto3.client('s3') for bucket in buckets: s3_client_.create_bucket(Bucket=bucket) yield s3_client_ s3_resource = boto3.resource('s3') for bucket in buckets: bucket = s3_resource.Bucket(bucket) bucket.objects.all().delete() bucket.delete() @pytest.fixture def uploaded_releases(s3_client, config_test): keys = [ 'A10301A0000935334X_20201116181657943/A10301A0000935334X.xml', 'A10301A0001138785R_20201116181529203/A10301A0001138785R.xml', 'A10301A0001267574Q_20201116181542913/A10301A0001267574Q.xml', 'acknowledgements/A10301A0009727524S.xml', 'acknowledgements_test/A10301A0009727524S.xml' ] content = 'xml_content' for key in keys: s3_client.put_object(Bucket=config_test.feed_bucket, Key=key, Body=content) @pytest.fixture def validator(payload_test, s3_client): schema_path = os.path.join(FIXTURES_PATH, 'validation', 'schema', '2021-04-20.xsd') with smart_open.open(schema_path) as file_: s3_client.put_object( Body=file_.read(), Bucket=payload_test.config_bucket, Key='schemas/ddex/release_message/v1/2021-04-20.xsd', ) return Validator( logger=mock.Mock(), payload=payload_test, s3_client=s3_client, ) @pytest.fixture def s3_service(s3_client, payload_test, config_test): return S3Service( logger=mock.Mock(), payload=payload_test, config=config_test, s3_client=s3_client ) @pytest.fixture def db_service(db): return DBService(logger=mock.Mock(), db_conn=db) @pytest.fixture def releases_service(config_test, payload_test, s3_service, db_service): return ReleasesService( logger=mock.Mock(), config=config_test, payload=payload_test, s3_service=s3_service, db_service=db_service, ) @pytest.fixture def release_service(payload_test, config_test, db_service, validator, s3_service): upload_local_releases(s3_service, config_test.feed_bucket) return ReleaseService( logger=mock.Mock(), release_title='valid_release', payload=payload_test, config=config_test, db_service=db_service, validator=validator, s3_service=s3_service, ) @pytest.fixture def unit_of_work(db): unit_of_work = slz.UnitOfWork( **{ 'unit_of_work_code': 'ddex-20210427-sme-release_message-v1', 'reprocess_id': '', 'report_date': datetime(2021, 4, 27), 'report_id': 1, 'licensor_id': 1, 'version': 'v1', 'activity_status': 'NOT_IN_PROGRESS', 'completeness_status': 'ACTIVE', 'next_run_at': '2020-04-27 01:06:09.715337', 'created_at': '2020-04-027 01:00:58.806272', 'last_updated_at': '2020-04-27 08:20:58.481813', 'is_force_complete': False, 'priority': 5 } ) db.session.add(unit_of_work) db.session.flush() yield unit_of_work @pytest.fixture def content_status(db, unit_of_work): content_status = slz.ContentStatus( **{ 'unit_of_work_id': unit_of_work.unit_of_work_id, 'context': 'A10301A0000935334X_20201116181657943', 'content_status': 'ACTIVE', 'content_name': 'ACK_A10301A00042689714_20210514115608363.xml', 'failure_count': 0, 'latest_job_id': 'ddex-20210427-sme-release_message-v1_26640_20210428T12.14.07', 'created_at': '2021-04-07 12:30:17', 'last_checked_at': '2021-04-09 12:30:17', 'metadata_process_status': 'NOT_QUEUED', } ) db.session.add(content_status) db.session.flush() yield content_status