import base64 import hashlib import os from abc import ABCMeta, abstractmethod from datetime import datetime, timedelta, timezone from operator import attrgetter from typing import Any import httpx import jwt import orjson from retry import retry from sqlalchemy.engine.row import Row from dapd_api_scraper.config import Config from dapd_api_scraper.etl_models import ( AppleMusicAlbum, AppleMusicArtist, AppleMusicPlaylist, AppleMusicTrack, SpotifyAlbum, SpotifyArtist, SpotifyPlaylist, SpotifyTrack, ) from dapd_api_scraper.etldb_models import DimAlbum, DimArtist, DimPlaylist, DimTrack from dapd_api_scraper.workflowdb_models import Album, Artist, Playlist, Track class Entity(metaclass=ABCMeta): chunk_size: int = 1 threads_count: int = 1 db_get_limit: int = 1000 is_removed_expired_ttl_days = 7 token_exp: datetime = datetime(1970, 1, 1, tzinfo=timezone.utc) token: str api_timeout: httpx.Timeout | int = httpx.Timeout(60, connect=60, read=60, write=60, pool=60) data_source: str api_url: str api_auth_url: str workflowdb_model: Any etldb_model: Any etl_model: Any entity_path: str entity_type: str def __init__(self, config: Config) -> None: self.config = config self.threads_count = int( os.getenv( f"{self.data_source}_{self.entity_type}_threads_count", str(self.threads_count), ) ) self.chunk_size = int( os.getenv( f"{self.data_source}_{self.entity_type}_chunk_size", str(self.chunk_size), ) ) self.is_removed_expired_ttl_days = int( os.getenv( f"{self.data_source}_{self.entity_type}_is_removed_expired_ttl_days", str(self.is_removed_expired_ttl_days), ) ) @abstractmethod def get_api_path_for_records(self, records: list[Row]) -> str: ... @abstractmethod def get_auth_headers(self) -> dict[str, str]: ... @abstractmethod def get_record_from_api_response(self, record: Row, data: Any) -> dict[str, Any] | None: ... def get_tracks_from_data(self, *args, **kwargs) -> list[Any]: return [] def get_albums_from_data(self, *args, **kwargs) -> list[Any]: return [] def get_artists_from_data(self, *args, **kwargs) -> list[Any]: return [] class SpotifyEntity(Entity): data_source = "spotify" api_url = "https://api.spotify.com/v1" api_auth_url = "https://accounts.spotify.com/api/token" def get_api_path_for_records(self, records: list[Row]) -> str: return self.entity_path + "?ids=" + ",".join(map(attrgetter("id"), records)) @retry(delay=5) def get_auth_headers(self) -> dict[str, str]: now = datetime.now(timezone.utc) if self.token_exp > now: return {"Authorization": f"Bearer {self.token}"} client_id = self.config.spotify_credentials["client_id"] client_secret = self.config.spotify_credentials["client_secret"] encoded = base64.b64encode(f"{client_id}:{client_secret}".encode("ascii")) headers = {"Authorization": f"Basic {encoded.decode()}"} auth_response = httpx.post( self.api_auth_url, data={"grant_type": "client_credentials"}, headers=headers, ) auth_response.raise_for_status() data = orjson.loads(auth_response.content) expires_in = data["expires_in"] access_token = data["access_token"] self.token_exp = now + timedelta(seconds=expires_in) self.token = access_token return {"Authorization": f"Bearer {self.token}"} def get_entities_ids_from_response(self, data: Any) -> list[str | int]: return [entity["id"] for entity in data.get(self.entity_type + "s", []) if entity] def get_record_from_api_response(self, record: Row, data: Any) -> dict[str, Any] | None: entity: dict[str, Any] for entity in data.get(self.entity_path, []): if entity and entity["id"] == record.id: return entity return None class SpotifyPlaylistEntity(SpotifyEntity): chunk_size = 1 threads_count = 16 is_removed_expired_ttl_days = 1 workflowdb_model = Playlist etldb_model = DimPlaylist etl_model = SpotifyPlaylist entity_path = "playlists" entity_type = "playlist" tracks_ttl_minutes = 60 * 24 * 365 albums_ttl_minutes = 60 * 24 * 365 artists_ttl_minutes = 60 * 24 def get_api_path_for_records(self, records: list[Row]) -> str: if len(records) != 1: raise Exception("Playlist records length should be 1") return f"/{self.entity_path}/{records[0].id}" def get_entities_ids_from_response(self, data: dict[str, Any]) -> list[str | int | Any]: if data: return [data.get("id")] return [] def get_record_from_api_response(self, record: Row, data: dict[str, Any]) -> dict[str, Any]: return data def get_tracks_from_data(self, records: list[Row], data): if not data or records[0].snapshot_id == data.get("snapshot_id"): return [] data_source = records[0].data_source storefront = records[0].storefront tracks = data.get("tracks", {}).get("items", []) tracks = tracks or [] for track in tracks: if isinstance(track.get("track"), dict): track.update(track.get("track")) tracks = [ { "id": track["id"], "storefront": storefront, "data_source": data_source, "ttl_minutes": self.tracks_ttl_minutes, } for track in tracks if track and track.get("id") ] return tracks def get_albums_from_data(self, records: list[Row], data) -> list[dict[str, Any]]: if not data or records[0].snapshot_id == data.get("snapshot_id"): return [] data_source = records[0].data_source storefront = records[0].storefront tracks = data.get("tracks", {}).get("items", []) tracks = tracks or [] for track in tracks: if isinstance(track.get("track"), dict): track.update(track.get("track")) albums = [track.get("album") for track in tracks if track] albums = [ { "id": album["id"], "storefront": storefront, "data_source": data_source, "ttl_minutes": self.albums_ttl_minutes, } for album in albums if album and album.get("id") ] return albums def get_artists_from_data(self, records: list[Row], data) -> list[dict[str, Any]]: if not data or records[0].snapshot_id == data.get("snapshot_id"): return [] data_source = records[0].data_source storefront = records[0].storefront tracks = data.get("tracks", {}).get("items", []) tracks = tracks or [] for track in tracks: if isinstance(track.get("track"), dict): track.update(track.get("track")) artists = [track.get("artists") for track in tracks if track and track.get("artists")] artists = [artist for subartists in artists for artist in subartists] artists = [ { "id": artist["id"], "storefront": storefront, "data_source": data_source, "ttl_minutes": self.artists_ttl_minutes, } for artist in artists if artist and artist.get("id") ] return artists class SpotifyTrackEntity(SpotifyEntity): chunk_size = 50 threads_count = 1 workflowdb_model = Track etldb_model = DimTrack etl_model = SpotifyTrack entity_path = "tracks" entity_type = "track" class SpotifyAlbumEntity(SpotifyEntity): chunk_size = 20 threads_count = 1 workflowdb_model = Album etldb_model = DimAlbum etl_model = SpotifyAlbum entity_path = "albums" entity_type = "album" class SpotifyArtistEntity(SpotifyEntity): chunk_size = 50 threads_count = 2 workflowdb_model = Artist etldb_model = DimArtist etl_model = SpotifyArtist entity_path = "artists" entity_type = "artist" class AppleMusicEntity(Entity): data_source = "apple_music" api_url = "https://api.music.apple.com/v1/catalog" api_key_ttl_seconds = 180 * 24 * 60 * 60 def get_api_path_for_records(self, records: list[Row]) -> str: storefront = records[0].storefront query = [] for record in records: query.append(f"ids={record.id}") compiled_query = "&".join(query) return f"/{storefront}{self.entity_path}?{compiled_query}" def get_auth_headers(self) -> dict[str, str]: now = datetime.now(timezone.utc) if self.token_exp > now: return {"Authorization": f"Bearer {self.token}"} time_expired = now + timedelta(seconds=self.api_key_ttl_seconds) headers = { "alg": "ES256", "kid": self.config.apple_music_credentials["APPLE_MUSIC_KEY_ID"], } payload = { "iss": self.config.apple_music_credentials["APPLE_MUSIC_TEAM_ID"], "iat": int(now.timestamp()), "exp": int(time_expired.timestamp()), } token = jwt.encode( payload, self.config.apple_music_credentials["APPLE_MUSIC_SECRET_KEY"], algorithm="ES256", headers=headers, ) self.token_exp = time_expired self.token = token return {"Authorization": f"Bearer {self.token}"} def get_entities_ids_from_response(self, data: Any) -> list[str | int]: if data: return [entity["id"] for entity in data.get("data", []) if entity] return [] def get_record_from_api_response(self, record: Row, data: Any) -> dict[str, Any] | None: entity: dict[str, Any] for entity in data["data"]: if entity and entity["id"] == record.id: return entity return None class AppleMusicPlaylistEntity(AppleMusicEntity): chunk_size = 25 threads_count = 8 is_removed_expired_ttl_days = 1 workflowdb_model = Playlist etldb_model = DimPlaylist etl_model = AppleMusicPlaylist entity_path = "/playlists" entity_type = "playlist" tracks_ttl_minutes = 60 * 24 * 365 def get_tracks_from_data(self, records: list[Row], data) -> list[dict[str, Any]]: if not data: return [] data_source = records[0].data_source storefront = records[0].storefront playlists = data["data"] tracks = [playlist["relationships"]["tracks"]["data"] for playlist in playlists] tracks = [track for subtracks in tracks for track in subtracks] tracks = [ { "id": track["id"], "storefront": storefront, "data_source": data_source, "ttl_minutes": self.tracks_ttl_minutes, } for track in tracks ] return tracks def get_record_from_api_response(self, record: Row, data: Any): for entity in data["data"]: if entity and entity["id"] == record.id: entity["snapshot_id"] = hashlib.sha256( str( [track["id"] for track in entity["relationships"]["tracks"]["data"]] ).encode() ).hexdigest() return entity return None class AppleMusicTrackEntity(AppleMusicEntity): chunk_size = 100 threads_count = 1 workflowdb_model = Track etldb_model = DimTrack etl_model = AppleMusicTrack entity_path = "/songs" entity_type = "track" albums_ttl_minutes = 60 * 24 * 365 artists_ttl_minutes = 60 * 24 * 365 def get_albums_from_data(self, records: list[Row], data) -> list[dict[str, Any]]: if not data: return [] data_source = records[0].data_source storefront = records[0].storefront tracks = data["data"] albums = [track["relationships"]["albums"]["data"] for track in tracks] albums = [album for subalbums in albums for album in subalbums] albums = [ { "id": album["id"], "storefront": storefront, "data_source": data_source, "ttl_minutes": self.albums_ttl_minutes, } for album in albums ] return albums def get_artists_from_data(self, records: list[Row], data) -> list[dict[str, Any]]: if not data: return [] data_source = records[0].data_source storefront = records[0].storefront tracks = data["data"] artists = [track["relationships"]["artists"]["data"] for track in tracks] artists = [artist for subartists in artists for artist in subartists] artists = [ { "id": artist["id"], "storefront": storefront, "data_source": data_source, "ttl_minutes": self.artists_ttl_minutes, } for artist in artists ] return artists class AppleMusicAlbumEntity(AppleMusicEntity): chunk_size = 100 threads_count = 1 workflowdb_model = Album etldb_model = DimAlbum etl_model = AppleMusicAlbum entity_path = "/albums" entity_type = "album" class AppleMusicArtistEntity(AppleMusicEntity): chunk_size = 25 threads_count = 1 workflowdb_model = Artist etldb_model = DimArtist etl_model = AppleMusicArtist entity_path = "/artists" entity_type = "artist"