import logging import os from time import sleep import boto3 import pytest from boto3_type_annotations.s3 import Client as S3Client from db_schema.schemas.admin import AuditLog from db_schema.schemas.apps import ContentStatus as AppsContentStatus from db_schema.schemas.apps import DatabricksExecution from db_schema.schemas.apps import FailureLog as AppsFailureLog from db_schema.schemas.apps import UnitOfWork as AppsUnitOfWork from db_schema.schemas.slz import ContentStatus, Report, Snapshot, UnitOfWork from moto import mock_s3, mock_sqs from smelog.entities import LoggerConfig from smelog.factory import LoggerFactory from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker from delphi_slz_admin.app_entry import setup_app from delphi_slz_admin.const import APP_NAME, VERSION test_engine = create_engine(os.getenv('DB_URL')) TestSession = sessionmaker(bind=test_engine) logger = None # pylint: disable=invalid-name @pytest.fixture(scope='session', autouse=True) def db(): test_session = TestSession(autocommit=True, autoflush=True) # Simple hack for waiting migrations to be completed while True: result = test_session.execute( # pylint: disable=no-member 'SELECT id FROM databasechangelog ORDER BY dateexecuted DESC LIMIT 1' ) if result.first(): break sleep(1) yield test_session @pytest.fixture(scope='function') def clean_db(db): # pylint: disable=redefined-outer-name yield for model in [AuditLog, ContentStatus, UnitOfWork]: db.query(model).delete() @pytest.fixture(scope='session', autouse=True) def logger_test(): global logger # pylint: disable=global-statement,invalid-name, global-variable-not-assigned log_config = LoggerConfig( name=APP_NAME, version=VERSION, level=logging.DEBUG, environment='test', is_local=True, ) return LoggerFactory(log_config).get_logger(APP_NAME) @pytest.fixture def app(): # pylint: disable=redefined-outer-name app = setup_app() return app @pytest.fixture def unit_of_work(db): # pylint: disable=redefined-outer-name report = db.query(Report).filter(Report.report_name == 'dim_track_participant').one() unit_of_work = UnitOfWork( **{ 'unit_of_work_code': 'gras-20200427-sme-dim_track_participant-v1', 'reprocess_id': '', 'report_date': '2020-04-27', 'report_id': report.report_id, 'licensor_id': 1, 'version': 'v1', 'activity_status': 'NOT_IN_PROGRESS', 'completeness_status': 'ACTIVE', 'next_run_at': '2020-04-28 01:06:09.715337', 'created_at': '2020-04-28 01:00:58.806272', 'last_updated_at': '2020-05-27 08:20:58.481813', 'is_force_complete': True, 'priority': 5 } ) db.add(unit_of_work) db.flush() yield unit_of_work db.delete(unit_of_work) db.flush() @pytest.fixture def content_status(db, unit_of_work): # pylint: disable=redefined-outer-name content_status = ContentStatus( **{ 'unit_of_work_id': unit_of_work.unit_of_work_id, 'context': 'dim_track_participant', 'content_status': 'ON_HOLD', 'content_name': 'gras_20200427_dim_track_participant.csv', 'content_size': 565118579, 'failure_count': 0, 'latest_job_id': '14260_2020-04-28T02.00.58', 'created_at': '2020-04-27 14:31:08.518350', 'completed_at': None, 'last_checked_at': '2020-04-27 15:39:33.622070', 'sub_content': 'null', 'record_count': 9188557, 'hash': '0265bed78e2fdccce1db1d4a28d926cf', 'metadata_process_status': 'COMPLETED', 'metadata_process_started_at': '2020-04-27 15:42:01.304492', 'metadata_process_completed_at': '2020-04-27 15:45:51.944453', 'meta_data': { 'version': '2.0.0', 'queues': ['https://queue.amazonaws.com/123456789012/arn-1'], 'message': { 'uow_id': 'uow-id', 'unit_of_work_id': 123, 'compressed_path': 's3://sme-archive/sme/v1/report.txt.gz', 'decompressed_paths': 's3://sme-decompressed/sme/v1/report.txt', 'content_name': 'content-name', 'context': 'US', 'optional_config': {} } } } ) db.add(content_status) db.flush() yield content_status db.delete(content_status) db.flush() @pytest.fixture def apps_unit_of_work(db): # pylint: disable=redefined-outer-name apps_unit_of_work = AppsUnitOfWork( **{ 'unit_of_work_code': 'spotify-20140201-streams_artist_playlist_date_day', 'report_date': '2014-02-01', 'report_id': 5, 'activity_status': 'NOT_IN_PROGRESS', 'completeness_status': 'FAILED', 'next_run_at': '2020-08-20 14:44:48.114833', 'created_at': '2020-07-15 08:17:59.330130', 'last_updated_at': '2020-08-20 14:44:48.114836', 'failure_count': 0, 'latest_job_id': 'spotify-20140201-streams_day_1130_20200820T14.44.48', 'latest_content_status_timestamp': '2020-07-15 08:17:59.330130', 'latest_job_state': None, 'priority': 9, 'unit_of_work_type': 'BACKFILL', 'data_source': 'SLZ', } ) db.add(apps_unit_of_work) db.flush() yield apps_unit_of_work db.delete(apps_unit_of_work) db.flush() @pytest.fixture def dbx_execution(db, apps_unit_of_work): # pylint: disable=redefined-outer-name dbx_execution = DatabricksExecution( **{ 'unit_of_work_id': apps_unit_of_work.unit_of_work_id, 'dbx_job_id': 164, 'dbx_run_id': 7608, 'contexts': '[]', 'status': 'COMPLETE', 'started_at': '2020-03-18 12:12:06.840446', 'completed_at': '2020-03-18 12:14:34.423176', 'dbx_job_expired_at': '2020-03-18 13:12:34.423176', 'sf_execution_name': 'amazonadsupported-20200309-stream_track2_2020-03-19T04.29.10' } ) db.add(dbx_execution) db.flush() yield dbx_execution db.delete(dbx_execution) db.flush() @pytest.fixture def apps_failure_log(db, apps_unit_of_work): # pylint: disable=redefined-outer-name apps_failure_log = AppsFailureLog( **{ 'unit_of_work_id': apps_unit_of_work.unit_of_work_id, 'job_id': 'some_job_id', 'failure_code': None, 'failure_description': 'foo_desc', 'created_at': '2020-03-18 11:08:12.371053' } ) db.add(apps_failure_log) db.flush() yield apps_failure_log db.delete(apps_failure_log) db.flush() @pytest.fixture def apps_content_status(db, apps_unit_of_work): # pylint: disable=redefined-outer-name apps_content_status = AppsContentStatus( **{ 'unit_of_work_id': apps_unit_of_work.unit_of_work_id, 'licensor_id': 1, 'report_id': 8, 'content_name': 'streams_20200317_PS.gz', 'latest_run_id': 8736, 'context': 'PS', 'created_at': '2020-04-03 12:08:04.611202' } ) db.add(apps_content_status) db.flush() yield apps_content_status db.delete(apps_content_status) db.flush() @pytest.fixture def snapshot(db, content_status): # pylint: disable=redefined-outer-name snapshot = Snapshot( **{ 'content_status_id': content_status.content_status_id, 'file_name': content_status.content_name, 'hash': '7c00ff0338fcbc78d6a4487fda47c608', 'created_at': '2020-04-03 12:08:04.611202' } ) db.add(snapshot) db.flush() yield snapshot db.delete(snapshot) db.flush() @pytest.fixture(scope='function', autouse=True) def aws_credentials(): os.environ['AWS_ACCESS_KEY_ID'] = 'testing' os.environ['AWS_SECRET_ACCESS_KEY'] = 'testing' os.environ['AWS_SECURITY_TOKEN'] = 'testing' os.environ['AWS_SESSION_TOKEN'] = 'testing' os.environ['AWS_DEFAULT_REGION'] = 'us-east-1' boto3.setup_default_session( aws_access_key_id='testing', aws_secret_access_key='testing', aws_session_token='testing', region_name='us-east-1', ) @pytest.fixture(scope='function') def sqs(aws_credentials): # pylint: disable=redefined-outer-name,unused-argument with mock_sqs(): sqs = boto3.client('sqs') sqs.create_queue(QueueName='arn-1') yield sqs @pytest.fixture def sqs_queue_arn(): return { 'arn-1': 'https://queue.amazonaws.com/123456789012/arn-1', 'arn-2': 'https://queue.amazonaws.com/123456789012/arn-2' } # pylint: disable=redefined-outer-name,unused-argument,invalid-name @pytest.fixture def s3(aws_credentials): with mock_s3(): s3_client: S3Client = boto3.client('s3') for bucket in ['bucket_archive', 'bucket_decompressed']: s3_client.create_bucket(Bucket=bucket) yield s3_client