# pylint: disable=redefined-outer-name,unused-argument,invalid-name import os from time import sleep import boto3 import pytest from boto3_type_annotations.s3 import Client as S3Client from dapd_db_schema.schemas.etl import DimDsp, DimMarket, DimPlaylist, DimTrack from dapd_db_schema.schemas.uow_meta import AuditLog, UnitOfWork, UnitOfWorkStatusEnum from moto import mock_s3 from sqlalchemy import create_engine from sqlalchemy.orm import Session, sessionmaker from dapd_exporter_lambda.services.config import ConfigService @pytest.fixture(scope='function') def db(): if os.getenv('IS_INTEGRATION_TESTING') and os.getenv('DB_URL'): test_engine = create_engine(os.getenv('DB_URL')) test_session_class = sessionmaker(bind=test_engine) test_session = test_session_class(autocommit=True, autoflush=True) # Simple hack for waiting migrations to be completed while True: result = test_session.execute( # pylint: disable=no-member 'SELECT id FROM databasechangelog ORDER BY dateexecuted DESC LIMIT 1' ) if result.first(): break sleep(1) yield test_session else: yield @pytest.fixture(scope='function', autouse=True) def clean_etl_db(db): yield if os.getenv('IS_INTEGRATION_TESTING'): for model in [ DimTrack, DimPlaylist, DimMarket, DimDsp, UnitOfWork, AuditLog, ]: db.query(model).delete() @pytest.fixture def dim_dsp(db: Session): instance = DimDsp(dsp_name='apple_music') db.add(instance) db.flush() yield instance @pytest.fixture def dim_market(db: Session): instance = DimMarket( **{ 'market_id': 1, 'market_code': 'global', 'market_name': 'foo_name', 'market_full_name': None, 'territory_type_id': None, 'parent_market_id': None, 'created_at': '2020-10-30 09:51:33.191551', 'updated_at': '2020-10-30 09:51:33.191000' } ) db.add(instance) db.flush() yield instance @pytest.fixture def dim_playlist(db: Session, dim_dsp: DimDsp, dim_market: DimMarket): instance = DimPlaylist( **{ 'dsp_id': dim_dsp.dsp_id, 'market_id': dim_market.market_id, 'playlist_id': '0076R65zbUgOgxM5tzWtsR', 'uri': 'spotify:playlist:0076R65zbUgOgxM5tzWtsR', 'user_id': 'pd_etldb_owner', 'type': None, 'is_public': False, 'is_personalized': False, 'collect_historical_data': False, 'updated_at': '2020-10-15 05:42:24.000000', 'created_at': '2020-10-21 22:52:53.335658' } ) db.add(instance) db.flush() yield instance @pytest.fixture def dim_track(db: Session, dim_dsp: DimDsp, dim_market: DimMarket): instance = DimTrack( **{ 'dsp_id': dim_dsp.dsp_id, 'market_id': dim_market.market_id, 'track_id': '2dNxQWaUJp1FWa7vhcI597', 'track_name': '062 - Spuk am Himmel - Teil 02', 'track_duration': 104, 'isrc': 'DEC711904690', 'audio_features': None, 'composer': None, 'upc': None, 'updated_at': '2020-10-24 22:11:00.385252', 'created_at': '2020-10-24 22:11:00.385252', 'release_date': '2020-10-24' } ) db.add(instance) db.flush() yield instance @pytest.fixture def unit_of_work(db: Session): instance = UnitOfWork( project='dapd', fact='fact_artist_followers', dimensions=['dim_playlist'], status=UnitOfWorkStatusEnum.EXPORT_IN_PROGRESS, version='v1', first_fact_id=0, last_fact_id=0, ) db.add(instance) db.flush() yield instance @pytest.fixture def config(): os.environ['DECIBEL_DB_SECRET_KEY'] = 'decibel_db_secret_key' os.environ['DECIBEL_ETL_DB_SECRET_KEY'] = 'decibel_db_secret_key' os.environ['DAPD_APPLE_DB_SECRET_KEY'] = 'dapd_apple_db_secret_key' os.environ['DAPD_APPLE_ETL_DB_SECRET_KEY'] = 'dapd_apple_db_secret_key' os.environ['DIFF_FLOW_DBX_MAX_RETRIES'] = '10' os.environ['DIFF_FLOW_MAX_CHUNK_SIZE'] = '100' env = 'test' event = { 'project': 'dapd_apple', } config_service = ConfigService(env, event) yield config_service.load_config() @pytest.fixture def config_for_artist(): os.environ['DECIBEL_DB_SECRET_KEY'] = 'decibel_db_secret_key' env = 'test' event = { 'fact': 'fact_artist_followers', 'dimensions': ['dim_artist'], 'project': 'dapd', } config_service = ConfigService(env, event) yield config_service.load_config() @pytest.fixture(scope='session') def aws_credentials(): os.environ['AWS_ACCESS_KEY_ID'] = 'testing' os.environ['AWS_SECRET_ACCESS_KEY'] = 'testing' os.environ['AWS_SECURITY_TOKEN'] = 'testing' os.environ['AWS_SESSION_TOKEN'] = 'testing' os.environ['AWS_DEFAULT_REGION'] = 'us-east-2' @pytest.fixture def s3(aws_credentials): with mock_s3(): try: del os.environ['AWS_DEFAULT_REGION'] except KeyError: pass s3_client: S3Client = boto3.client('s3') for bucket in [ 'test-delphi-public-data-etl-export', 'test-delphi-configs', 'test-delphi-decibel' ]: s3_client.create_bucket(Bucket=bucket) yield s3_client s3_resource = boto3.resource('s3') for bucket in [ 'test-delphi-public-data-etl-export', 'test-delphi-configs', 'test-delphi-decibel' ]: bucket = s3_resource.Bucket(bucket) bucket.objects.all().delete() bucket.delete()