# pylint: disable=redefined-outer-name,unused-argument,invalid-name import os from time import sleep import boto3 import pytest from boto3_type_annotations.s3 import Client as S3Client from dapd_db_schema.schemas.etl import ( DimArtist, DimDsp, DimMarket, DimPlaylist, DimTrack, DimTrackArtist, FactArtistFollowers, PlaylistOwner, PlaylistOwnerCategory, ) from dapd_db_schema.schemas.uow_meta import AuditLog, UnitOfWork, UnitOfWorkStatusEnum from moto import mock_s3 from sqlalchemy import create_engine from sqlalchemy.orm import Session, sessionmaker from dapd_exporter.services.config import ConfigService @pytest.fixture(scope='function') def db(): if os.getenv('IS_INTEGRATION_TESTING') and os.getenv('DB_URL'): test_engine = create_engine(os.getenv('DB_URL')) test_session_class = sessionmaker(bind=test_engine) test_session = test_session_class(autocommit=True, autoflush=True) # Simple hack for waiting migrations to be completed while True: result = test_session.execute( # pylint: disable=no-member 'SELECT id FROM databasechangelog ORDER BY dateexecuted DESC LIMIT 1' ) if result.first(): break sleep(1) yield test_session else: yield @pytest.fixture(scope='function') def db_engine(): if os.getenv('IS_INTEGRATION_TESTING') and os.getenv('DB_URL'): test_engine = create_engine(os.getenv('DB_URL')) yield test_engine else: yield @pytest.fixture(scope='function', autouse=True) def clean_db(db): yield if os.getenv('IS_INTEGRATION_TESTING'): for model in [ DimTrackArtist, FactArtistFollowers, DimArtist, DimTrack, DimPlaylist, DimMarket, DimDsp, AuditLog, UnitOfWork, PlaylistOwner, PlaylistOwnerCategory ]: db.query(model).delete() @pytest.fixture def dim_dsp(db: Session): instance = DimDsp(dsp_name='apple_music') db.add(instance) db.flush() yield instance @pytest.fixture def dim_market(db: Session): instance = DimMarket( **{ 'market_id': 1, 'market_code': 'global', 'market_name': 'foo_name', 'market_full_name': None, 'territory_type_id': None, 'parent_market_id': None, 'created_at': '2020-10-30 09:51:33.191551', 'updated_at': '2020-10-30 09:51:33.191000' } ) db.add(instance) db.flush() yield instance @pytest.fixture def playlist_owner_category(db: Session): instance = PlaylistOwnerCategory(**{ 'id': 1, 'name': 'Warner', }) db.add(instance) db.flush() yield instance @pytest.fixture def playlist_owner( db: Session, dim_dsp: DimDsp, dim_market: DimMarket, playlist_owner_category: PlaylistOwnerCategory ): instance = PlaylistOwner( **{ 'dsp_id': dim_dsp.dsp_id, 'market_id': dim_market.market_id, 'username': 'digster.pt', 'display_name': 'Digster Portugal', 'category_id': playlist_owner_category.id, } ) db.add(instance) db.flush() yield instance @pytest.fixture def dim_playlist( db: Session, dim_dsp: DimDsp, dim_market: DimMarket, playlist_owner: PlaylistOwner ): instance = DimPlaylist( **{ 'dsp_id': dim_dsp.dsp_id, 'market_id': dim_market.market_id, 'playlist_id': '0076R65zbUgOgxM5tzWtsR', 'uri': 'spotify:playlist:0076R65zbUgOgxM5tzWtsR', 'playlist_owner_username': playlist_owner.username, 'type': None, 'is_public': False, 'is_personalized': False, 'collect_historical_data': False, 'updated_at': '2020-10-15 05:42:24.000000', 'created_at': '2020-10-21 22:52:53.335658' } ) db.add(instance) db.flush() yield instance @pytest.fixture def dim_track(db: Session, dim_dsp: DimDsp, dim_market: DimMarket): instance = DimTrack( **{ 'dsp_id': dim_dsp.dsp_id, 'market_id': dim_market.market_id, 'track_id': '2dNxQWaUJp1FWa7vhcI597', 'track_name': '062 - Spuk am Himmel - Teil 02', 'track_duration': 104, 'isrc': 'DEC711904690', 'audio_features': None, 'composer': None, 'upc': None, 'updated_at': '2020-10-24 22:11:00.385252', 'created_at': '2020-10-24 22:11:00.385252', 'release_date': '2020-10-24' } ) db.add(instance) db.flush() yield instance @pytest.fixture def dim_artist(db: Session, dim_dsp: DimDsp, dim_market: DimMarket): instance = DimArtist( **{ 'dsp_id': dim_dsp.dsp_id, 'market_id': dim_market.market_id, 'artist_id': 'foo_id', 'artist_name': 'Foo Name', 'genres': None, 'image_path': None, 'artist_uri': 'foo_id_uri', 'is_blacklisted': False, 'created_at': '2020-10-30 09:49:05.606156', 'updated_at': '2020-10-30 09:49:05.606156', } ) db.add(instance) db.flush() yield instance @pytest.fixture def fact_artist_followers(db: Session, dim_artist: DimArtist): instance = FactArtistFollowers(id=100500, artist_id=dim_artist.artist_id, followers=100) db.add(instance) db.flush() yield instance @pytest.fixture(scope='function') def dim_track_artist(db: Session, dim_artist: DimArtist, dim_track: DimTrack): instance = DimTrackArtist( dsp_id=dim_track.dsp_id, track_id=dim_track.track_id, artist_id=dim_artist.artist_id, market_id=dim_track.market_id, artist_number=1, updated_at='2020-10-15 05:42:24.000000', created_at='2020-10-21 22:52:53.335658' ) db.add(instance) db.flush() yield instance @pytest.fixture def unit_of_work(db: Session): instance = UnitOfWork( project='dapd', fact='fact_artist_followers', dimensions=['dim_playlist', 'dim_track'], status=UnitOfWorkStatusEnum.EXPORT_IN_PROGRESS, version='v1', first_fact_id=0, last_fact_id=0, ) db.add(instance) db.flush() yield instance @pytest.fixture def unit_of_work_artist(db: Session): instance = UnitOfWork( project='dapd', fact='fact_artist_followers', dimensions=[], status=UnitOfWorkStatusEnum.EXPORT_IN_PROGRESS, version='v1', first_fact_id=100499, last_fact_id=100501, ) db.add(instance) db.flush() yield instance @pytest.fixture(scope='session') def config(): env = 'test' etl_db_secret_key = 'test' api_db_secret_key = 'test' workflow_db_secret_key = 'test' config_service = ConfigService( env, etl_db_secret_key, api_db_secret_key, workflow_db_secret_key ) yield config_service.load_config() @pytest.fixture(scope='session') def aws_credentials(): os.environ['AWS_ACCESS_KEY_ID'] = 'testing' os.environ['AWS_SECRET_ACCESS_KEY'] = 'testing' os.environ['AWS_SECURITY_TOKEN'] = 'testing' os.environ['AWS_SESSION_TOKEN'] = 'testing' os.environ['AWS_DEFAULT_REGION'] = 'us-east-2' @pytest.fixture def s3(aws_credentials): with mock_s3(): del os.environ['AWS_DEFAULT_REGION'] s3_client: S3Client = boto3.client('s3') s3_client.create_bucket(Bucket='test-delphi-public-data-etl-export') yield s3_client s3_resource = boto3.resource('s3') bucket = s3_resource.Bucket('test-delphi-public-data-etl-export') bucket.objects.all().delete() bucket.delete()