# pylint: disable=unused-argument,protected-access,too-many-arguments,redefined-outer-name import os import uuid from unittest import mock import boto3 import pytest from boto3_type_annotations.s3 import Client as S3Client from db_schema.postgres import connection from db_schema.schemas import slz from moto import mock_s3 from slz_storage.repository import Repository from spotipy import SpotifyClientCredentials from slz_spotify_charts_scraper.__main__ import LambdaContext from slz_spotify_charts_scraper.const import SpotifyChartRecurrenceType, SpotifyChartType from slz_spotify_charts_scraper.entities import Config, Payload from slz_spotify_charts_scraper.repository import ContentStatusRepository, SnapshotRepository from slz_spotify_charts_scraper.service import S3Service, SpotifyChartsService from slz_spotify_charts_scraper.validator import Validator from tests import FIXTURES_PATH from tests.entities import BucketPathsTuple @pytest.fixture(scope='session') def db(): pg = connection.get_session( host=os.environ.get('PG_HOST', '0.0.0.0'), port=os.environ.get('PG_PORT', 5432), db=os.environ.get('PG_DB', 'slz'), user=os.environ.get('PG_USER', 'admin'), password=os.environ.get('PG_PASSWORD', 'admin'), engine_params={'echo': True} ) yield pg @pytest.fixture(scope='function') def clean_db(db): yield for model in [slz.ContentFailureLog, slz.Snapshot, slz.ContentStatus, slz.UnitOfWork]: db.query(model).delete() db.commit() db.close() @pytest.fixture def config_test(): return Config( environment='test', rds_secret_key='rds_secret_key', sentry_secret_key='', spotify_charts_url_base='spotify-private-url-base', spotify_secret_key='', quarantine_bucket='quarantine_bucket', decompressed_bucket='decompressed_bucket', corrupted_bucket='corrupted_bucket', config_bucket='config_bucket', concurrency=1 ) @pytest.fixture def payload_test(): return Payload( uow_id='spotify-20210801-sme-charts_daily_regional-v1', dsp='spotify', report_type='charts_daily_regional', application='charts', version='v1', licensor='sme', report_date='2021-08-01', job_id='test_job', recurrence_type=SpotifyChartRecurrenceType.DAILY, chart_type=SpotifyChartType.REGIONAL, markets=['us'] ) @pytest.fixture def s3_client(config_test): with mock_s3(): buckets = [ config_test.config_bucket, config_test.quarantine_bucket, config_test.decompressed_bucket, config_test.corrupted_bucket ] s3_client_: S3Client = boto3.client('s3') for bucket in buckets: s3_client_.create_bucket(Bucket=bucket) yield s3_client_ s3_resource = boto3.resource('s3') for bucket in buckets: bucket = s3_resource.Bucket(bucket) bucket.objects.all().delete() bucket.delete() @pytest.fixture def s3_service(): return S3Service(logger=mock.Mock()) @pytest.fixture def lambda_context(): return LambdaContext(aws_request_id=uuid.uuid4().hex) @pytest.fixture def snapshot_repository(db, payload_test): return SnapshotRepository( logger=mock.Mock(), session=db, payload=payload_test, ) @pytest.fixture def slz_repository(db): return Repository( logger=mock.Mock(), pg_conn=db, ) @pytest.fixture def content_status_repository(payload_test, slz_repository): return ContentStatusRepository( logger=mock.Mock(), payload=payload_test, slz_pg_repository=slz_repository ) @pytest.fixture def unit_of_work(db): report = db.query(slz.Report).filter(slz.Report.report_name == 'charts_daily_regional').one() unit_of_work = slz.UnitOfWork( **{ 'unit_of_work_code': 'spotify-20210801-sme-charts_daily_regional-v1', 'reprocess_id': '', 'report_date': '2021-08-01', 'report_id': report.report_id, 'licensor_id': 1, 'version': 'v1', 'activity_status': 'NOT_IN_PROGRESS', 'completeness_status': 'ACTIVE', 'next_run_at': '2021-08-02 08:26:09.715337', 'created_at': '2021-08-01 01:00:58.806272', 'last_updated_at': '2021-08-01 08:20:58.481813', 'is_force_complete': False, 'priority': 5 } ) db.add(unit_of_work) db.flush() yield unit_of_work @pytest.fixture def content_status(db, unit_of_work): content_status = slz.ContentStatus( **{ 'unit_of_work_id': unit_of_work.unit_of_work_id, 'context': 'us', 'content_status': 'ACTIVE', 'content_name': 'charts_daily_regional_20210801_us.parquet', 'failure_count': 0, 'latest_job_id': 'test_job_id__charts_daily_regional', 'created_at': '2021-08-01 12:30:17', 'last_checked_at': '2021-08-02 12:30:17', 'metadata_process_status': 'NOT_QUEUED', } ) db.add(content_status) db.flush() yield content_status @pytest.fixture def content_failure_log(db, content_status): content_failure_log = slz.ContentFailureLog( **{ 'content_status_id': content_status.content_status_id, 'job_id': 'some_job_id', 'failure_code': None, 'failure_description': 'failure_description', 'created_at': '2021-08-01 14:07:13.570531' } ) db.add(content_failure_log) db.flush() yield content_failure_log @pytest.fixture def snapshot(db, content_status): snapshot = slz.Snapshot( **{ 'content_status_id': content_status.content_status_id, 'file_name': content_status.content_name, 'hash': '7c00ff0338fcbc78d6a4487fda47c608', 'created_at': '2021-08-01 12:08:04.611202' } ) db.add(snapshot) db.flush() yield snapshot @pytest.fixture def snapshot_of_hash(db, content_status, request): """Snapshot instance with indirect hash, intended for integration processing testing.""" snapshot = slz.Snapshot( **{ 'content_status_id': content_status.content_status_id, 'file_name': content_status.content_name, # This hash is expected for current Spotify response fixture. 'hash': request.param, 'created_at': '2021-08-01 12:08:04.611202' } ) db.add(snapshot) db.flush() yield snapshot db.delete(snapshot) db.flush() @pytest.fixture def validator(payload_test, config_test): return Validator( logger=mock.Mock(), config=config_test, payload=payload_test, ) @pytest.fixture def spotify_service( payload_test, config_test, s3_service, slz_repository, validator, content_status_repository, snapshot_repository, lambda_context ): return SpotifyChartsService( logger=mock.Mock(), payload=payload_test, config=config_test, s3_service=s3_service, credentials_client=mock.MagicMock(SpotifyClientCredentials), slz_pg_repository=slz_repository, validator=validator, snapshot_repository=snapshot_repository, content_status_repository=content_status_repository, lambda_context=lambda_context ) @pytest.fixture def spotify_service_stateless( payload_test, config_test, s3_service, lambda_context, ): return SpotifyChartsService( logger=mock.Mock(), payload=payload_test, config=config_test, s3_service=s3_service, credentials_client=mock.MagicMock(SpotifyClientCredentials), slz_pg_repository=mock.Mock(), validator=mock.Mock(), snapshot_repository=mock.Mock(), content_status_repository=mock.Mock(), lambda_context=lambda_context, ) @pytest.fixture def bucket_paths_stateless( config_test, spotify_service_stateless, payload_test ) -> BucketPathsTuple: return BucketPathsTuple( quarantine=spotify_service_stateless._get_directory_path( bucket=config_test.quarantine_bucket, payload=payload_test, ), corrupted=spotify_service_stateless._get_directory_path( bucket=config_test.corrupted_bucket, payload=payload_test, ), decompressed=spotify_service_stateless._get_directory_path( bucket=config_test.decompressed_bucket, payload=payload_test, ), ) @pytest.fixture def charts_page_content() -> bytes: response_path = os.path.join(FIXTURES_PATH, 'data', 'top200_daily_us_raw.json') with open(response_path, 'rb') as file_: return file_.read()