"""Tests for vendor_image_caching lambda.""" import json import uuid from datetime import datetime from typing import Callable import boto3 import mongomock import pytest import requests_mock from moto import mock_s3 from tests.data import ( get_album, get_album_tracks, get_artist, get_artist_albums, get_episode, get_playlist, get_playlist_images, get_search_results, get_track, get_user, ) from vendor_image_caching import config from vendor_image_caching import handler from vendor_image_caching import lambda_handler from vendor_image_caching.cache import mongodb, redis, cache_backend from vendor_image_caching.constants import CollectionName, CacheField from vendor_image_caching.logger import get_logger from vendor_image_caching.utils import compress, decompress def get_context(): """Get fake lambda context. """ class Context: aws_request_id = 1 return Context() def make_event(collection_name: str, record_id: str, created_at: str or datetime) -> str: """Generate single event. Args: collection_name: Collection name. record_id: Cache record ID. created_at: Cache record created at. Returns: Event as str. """ return json.dumps({ "collection": collection_name, "id": record_id, "created_at": created_at if isinstance(created_at, str) else created_at.strftime("%Y-%m-%d %H:%M:%S.%f") }, default=str) def get_events(collection_name: str, record_id: str, created_at: str or datetime) -> dict: """Get fake event. Args: collection_name: Collection name. record_id: Cache record ID. created_at: Cache record created at. Returns: Lambda fake event dict. """ return { "Records": [ { "body": make_event(collection_name, record_id, created_at) }, ] } def get_mongo_record(record_id: str, created_at: datetime, data: dict, compression: str) -> dict: """Generate mongo cache record. Args: record_id: Cache record ID. created_at Cache record created at. data: Record data. compression: Compression type. Returns: Cache record. """ return { CacheField.RECORD_ID: record_id, CacheField.DATA: compress(compression, data), CacheField.IMAGES_SAVED: None, CacheField.CREATED_AT: created_at, CacheField.COMPRESSION: compression, } def get_redis_value(record_id: str, created_at: datetime, data: dict, compression: str) -> dict: """Generate cache record. Args: record_id: Cache record ID. created_at Cache record created at. data: Record data. compression: Compression type. Returns: Cache record. """ return compress( config.REDIS_COMPRESSION_LIB, { CacheField.RECORD_ID: record_id, CacheField.DATA: data, CacheField.CREATED_AT: created_at.isoformat(timespec="milliseconds"), } ) def test_lambda_handler(mocker): test_context = get_context() test_events = get_events("test_collection", "test_record_id", datetime.now()) test_logger = get_logger(test_context.aws_request_id) mocked_handler = mocker.patch("vendor_image_caching.handler") lambda_handler(test_events, test_context) assert mocked_handler.call_args_list == [ mocker.call(test_logger, test_events, cache_backend) ] @pytest.mark.parametrize( "collection_name,data_getter,force_unique,record_id,compression", ( (CollectionName.TRACK, get_track, False, "track_id_1", "zstd"), (CollectionName.TRACK, get_track, True, "track_id_2", "zlib"), (CollectionName.ALBUM, get_album, False, "album_id_1", None), (CollectionName.ALBUM, get_album, True, "album_id_2", None), (CollectionName.ARTIST, get_artist, False, "artist_id_1", None), (CollectionName.ARTIST, get_artist, True, "artist_id_2", None), (CollectionName.PLAYLIST, get_playlist, False, "playlist_id_1", None), (CollectionName.PLAYLIST, get_playlist, True, "playlist_id_2", None), (CollectionName.USER, get_user, False, "user_id_1", None), (CollectionName.USER, get_user, True, "user_id_2", None), (CollectionName.SEARCH, get_search_results, False, "text_123_us", None), (CollectionName.SEARCH, get_search_results, True, "text_123_us", None), (CollectionName.CACHE, get_episode, False, "v1/episodes/_episode_id_1,id_2_us", "zlib"), (CollectionName.CACHE, get_episode, True, "v1/episodes/_episode_id_1,id_2_us", "zlib"), (CollectionName.CACHE, get_artist_albums, False, "v1/artists/artist1/albums", "zlib"), (CollectionName.CACHE, get_artist_albums, True, "v1/artists/artist2/albums_us", "zstd"), (CollectionName.ALBUM_TRACKS, get_album_tracks, False, "album1", "zstd"), (CollectionName.ALBUM_TRACKS, get_album_tracks, True, "album2_us", "zstd"), (CollectionName.CACHE, get_playlist_images, False, "v1/playlists/pl1/images_us", None), (CollectionName.CACHE, get_playlist_images, True, "v1/playlists/pl2/images", None), ), ) @mock_s3 @requests_mock.Mocker(kw="requests_mocker") @mongomock.patch(servers=((config.MONGODB_HOST, config.MONGODB_PORT),)) def test_handler_with_mongodb( collection_name: str, data_getter: Callable, force_unique: bool, record_id: str, compression: str, **kwargs ): """ Test for handler with mongo backend. """ config.FORCE_UNIQUE_IMAGE_NAME = force_unique requests_mocker = kwargs["requests_mocker"] created_at = datetime.utcnow() original_data, original_images = data_getter(True, record_id) image_data = b"resp" s3_resource = boto3.resource("s3", region_name=config.AWS_DEFAULT_REGION) s3_resource.create_bucket(Bucket=config.AWS_S3_BUCKET_NAME) for image in original_images: requests_mocker.register_uri("GET", image["url"], content=image_data) mongo_wrapped_client = mongodb.Client() mongo_wrapped_client.mongo_client[config.MONGODB_DATABASE][collection_name].insert_one( get_mongo_record(record_id, created_at, original_data, compression) ) logger = get_logger(str(uuid.uuid4())) handler(logger, get_events(collection_name, record_id, created_at), mongodb) modified_data, modified_images = data_getter(False, record_id) bucket = s3_resource.Bucket(config.AWS_S3_BUCKET_NAME) s3_files = list(bucket.objects.all()) assert len(s3_files) == len(modified_images) for image in modified_images: img_path = image["url"].replace(f"{config.AWS_IMAGE_STORAGE_BASE_URL}/", "") file_data = s3_resource.Object(config.AWS_S3_BUCKET_NAME, img_path).get()["Body"].read() assert file_data == image_data mongo_record = mongo_wrapped_client.mongo_client[config.MONGODB_DATABASE][collection_name].find_one({ CacheField.RECORD_ID: record_id, CacheField.CREATED_AT: created_at, CacheField.IMAGES_SAVED: True }) assert mongo_record is not None assert decompress(config.MONGODB_COMPRESSION_LIB, mongo_record["data"]) == modified_data mongo_wrapped_client.mongo_client[config.MONGODB_DATABASE][collection_name].drop() @pytest.mark.parametrize( "collection_name,data_getter,force_unique,record_id,compression", ( (CollectionName.TRACK, get_track, False, "track_id_1", "zstd"), (CollectionName.TRACK, get_track, True, "track_id_2", "zlib"), (CollectionName.ALBUM, get_album, False, "album_id_1", None), (CollectionName.ALBUM, get_album, True, "album_id_2", None), (CollectionName.ARTIST, get_artist, False, "artist_id_1", None), (CollectionName.ARTIST, get_artist, True, "artist_id_2", None), (CollectionName.PLAYLIST, get_playlist, False, "playlist_id_1", None), (CollectionName.PLAYLIST, get_playlist, True, "playlist_id_2", None), (CollectionName.USER, get_user, False, "user_id_1", None), (CollectionName.USER, get_user, True, "user_id_2", None), (CollectionName.SEARCH, get_search_results, False, "text_123_us", None), (CollectionName.SEARCH, get_search_results, True, "text_123_us", None), (CollectionName.CACHE, get_episode, False, "v1/episodes/_episode_id_1,id_2_us", "zlib"), (CollectionName.CACHE, get_episode, True, "v1/episodes/_episode_id_1,id_2_us", "zlib"), (CollectionName.CACHE, get_artist_albums, False, "v1/artists/artist1/albums", "zlib"), (CollectionName.CACHE, get_artist_albums, True, "v1/artists/artist2/albums_us", "zstd"), (CollectionName.ALBUM_TRACKS, get_album_tracks, False, "album1", "zstd"), (CollectionName.ALBUM_TRACKS, get_album_tracks, True, "album2_us", "zstd"), (CollectionName.CACHE, get_playlist_images, False, "v1/playlists/pl1/images_us", None), (CollectionName.CACHE, get_playlist_images, True, "v1/playlists/pl2/images", None), ), ) @mock_s3 @requests_mock.Mocker(kw="requests_mocker") def test_handler_with_redis( collection_name: str, data_getter: Callable, force_unique: bool, record_id: str, compression: str, **kwargs ): """ Test for handler with redis backend. """ config.FORCE_UNIQUE_IMAGE_NAME = force_unique requests_mocker = kwargs["requests_mocker"] redis_key = f"{redis.CACHE_KEY_PREFIX}:{config.REDIS_COMPRESSION_LIB}:{collection_name}:{record_id}" created_at = datetime.utcnow() original_data, original_images = data_getter(True, record_id) image_data = b"resp" s3_resource = boto3.resource("s3", region_name=config.AWS_DEFAULT_REGION) s3_resource.create_bucket(Bucket=config.AWS_S3_BUCKET_NAME) for image in original_images: requests_mocker.register_uri("GET", image["url"], content=image_data) redis_wrapped_client = redis.Client() redis_wrapped_client.redis_client.set( redis_key, get_redis_value(record_id, created_at, original_data, compression) ) logger = get_logger(str(uuid.uuid4())) handler(logger, get_events(collection_name, record_id, created_at), redis) modified_data, modified_images = data_getter(False, record_id) bucket = s3_resource.Bucket(config.AWS_S3_BUCKET_NAME) s3_files = list(bucket.objects.all()) assert len(s3_files) == len(modified_images) for image in modified_images: img_path = image["url"].replace(f"{config.AWS_IMAGE_STORAGE_BASE_URL}/", "") file_data = s3_resource.Object(config.AWS_S3_BUCKET_NAME, img_path).get()["Body"].read() assert file_data == image_data value = decompress(config.REDIS_COMPRESSION_LIB, redis_wrapped_client.redis_client.get(redis_key)) record = value.get(CacheField.DATA) assert record is not None assert record == modified_data redis_wrapped_client.redis_client.delete(redis_key)