import logging import os from datetime import datetime, timedelta, timezone from time import sleep import boto3 import pytest from db_schema.postgres.connection import Connection from db_schema.schemas import slz from freezegun import freeze_time from moto import mock_s3 from mypy_boto3_s3.client import S3Client from psycopg2.extras import DateTimeRange from smelog.entities import LoggerConfig from smelog.factory import LoggerFactory ARCHIVE_BUCKET_TEST = 'sme_test' DECOMPRESSED_BUCKET_TEST = 'sme_decompressed_test' @pytest.fixture(scope='function', autouse=True) def aws_credentials(): os.environ['AWS_ACCESS_KEY_ID'] = 'testing' os.environ['AWS_SECRET_ACCESS_KEY'] = 'testing' os.environ['AWS_SECURITY_TOKEN'] = 'testing' os.environ['AWS_SESSION_TOKEN'] = 'testing' os.environ['AWS_DEFAULT_REGION'] = 'us-east-1' boto3.setup_default_session( aws_access_key_id='testing', aws_secret_access_key='testing', aws_session_token='testing', region_name='us-east-1', ) @pytest.fixture def s3_client(aws_credentials): with mock_s3(): s3_client_: S3Client = boto3.client('s3') for bucket in [ARCHIVE_BUCKET_TEST, DECOMPRESSED_BUCKET_TEST]: s3_client_.create_bucket(Bucket=bucket) yield s3_client_ s3_resource = boto3.resource('s3') for bucket in [ARCHIVE_BUCKET_TEST, DECOMPRESSED_BUCKET_TEST]: bucket = s3_resource.Bucket(bucket) bucket.objects.all().delete() bucket.delete() @pytest.fixture def db() -> Connection: def creds_loader(): return dict( active_endpoint=os.environ.get('PG_HOST', '0.0.0.0'), port=os.environ.get('PG_PORT', 5432), database=os.environ.get('PG_DB', 'slz'), username=os.environ.get('PG_USER', 'admin'), password=os.environ.get('PG_PASSWORD', 'admin'), ) conn = Connection(credentials_loader=creds_loader, name='s3_storage_test', version='v1') # Simple hack for waiting migrations to be completed while True: result = conn.session.execute( # pylint: disable=no-member 'SELECT id FROM databasechangelog ORDER BY dateexecuted DESC LIMIT 1' ) if result.first(): break sleep(1) yield conn conn.session.rollback() for model in [ slz.UnitOfWorkMigrationLog, slz.ContentStatusMigrationLog, slz.ContentFailureLog, slz.ContentStatus, slz.UnitOfWork, ]: conn.session.query(model).delete() conn.session.commit() conn.disconnect() @pytest.fixture(scope='function') def db_licensors(db): return {item.licensor_name: item for item in db.session.query(slz.Licensor).all()} @pytest.fixture(scope='function') def db_reports(db): return {item.report_name: item for item in db.session.query(slz.Report).all()} @pytest.fixture(scope='function') def db_data_sources(db): return {item.data_source_name: item for item in db.session.query(slz.DataSource).all()} @pytest.fixture(scope='function') def logger_test(): log_config = LoggerConfig( name='TEST', version='1', level=logging.DEBUG, environment='test', is_local=True, ) return LoggerFactory(log_config).get_logger('TEST') @pytest.fixture(scope='function') def now_freezed(): with freeze_time(datetime(2021, 11, 24, 3, 21, 34, tzinfo=timezone.utc)): yield datetime.now().astimezone(timezone.utc) @pytest.fixture(scope='function') def unit_of_work_stubs(db_licensors, now_freezed, db_reports): return { 1: dict( unit_of_work_code='appreciationengine-20191124-sme-membersvisittotals-v1', reprocess_id='', licensor=db_licensors['sme'], report=db_reports['membersvisittotals'], report_date='2019-11-24', version='v1', activity_status=slz.ActivityStatusEnum.NOT_IN_PROGRESS.value, completeness_status=slz.CompletenessStatusEnum.ACTIVE.value, is_force_complete=False, priority=slz.UnitOfWorkPriorityEnum.DEFAULT.value, next_run_at=now_freezed + timedelta(minutes=15), created_at=now_freezed, last_updated_at=now_freezed, timeslot=DateTimeRange( datetime(year=2021, month=1, day=1, hour=1), datetime(year=2021, month=1, day=1, hour=4), ) ), 2: dict( unit_of_work_code='appreciationengine-20191124-sme-membersvisittotals-v1', reprocess_id='123', licensor=db_licensors['sme'], report=db_reports['membersvisittotals'], report_date='2019-11-24', version='v1', activity_status=slz.ActivityStatusEnum.NOT_IN_PROGRESS.value, completeness_status=slz.CompletenessStatusEnum.ACTIVE.value, is_force_complete=False, priority=slz.UnitOfWorkPriorityEnum.DEFAULT.value, next_run_at=now_freezed + timedelta(minutes=15), created_at=now_freezed, last_updated_at=now_freezed, timeslot=DateTimeRange( datetime(year=2021, month=1, day=1, hour=1), datetime(year=2021, month=1, day=1, hour=4), ) ), } @pytest.fixture(scope='function') def content_status_stubs(now_freezed): return { 1: dict( context='US', content_name='us.txt', content_status=slz.ContentStatusEnum.COMPLETE, created_at=now_freezed, failure_count=0, sub_content='{}', latest_job_id='test_job_id_1', metadata_process_status=slz.ContentMetadataStatusEnum.NOT_QUEUED.value, ), 2: dict( context='MX', content_name='mx.txt', content_status=slz.ContentStatusEnum.MISSING, created_at=now_freezed, failure_count=0, sub_content='{}', latest_job_id='test_job_id_1', metadata_process_status=slz.ContentMetadataStatusEnum.NOT_QUEUED.value, ), 3: dict( context='IT', content_name='it.txt', content_status=slz.ContentStatusEnum.COMPLETE, failure_count=0, created_at=now_freezed, sub_content='{}', latest_job_id='test_job_id_2', metadata_process_status=slz.ContentMetadataStatusEnum.NOT_QUEUED.value, ), }