from pathlib import PurePosixPath from pycountry import countries from slugify import slugify from config import ELASTIC_SEARCH_ENV N_A_VALUE = "N/A" N_A_SLUG = slugify(N_A_VALUE) AVAILABLE_COUNTRIES = tuple(slugify(country.alpha_2) for country in countries) + (N_A_SLUG,) ARTIST_SEARCH_INDEX_NAME = f"{ELASTIC_SEARCH_ENV}_artist_search" TRACK_SEARCH_INDEX_NAME = f"{ELASTIC_SEARCH_ENV}_track_search" ARTIST_SEARCH_DATA_TABLE_NAME = "artist_search_data" TRACK_SEARCH_DATA_TABLE_NAME = "track_search_data" SLUGIFY_PROPS = ("country", "genres") ARTIST_SEARCH_TOP_PROPS = ("id", "spotify_artist_id", "name", "country", "genres", "label", "artwork_url") TRACK_SEARCH_TOP_PROPS = ( "id", "country", "artwork_url", "isrc", "name", "artist_name", "cm_artist_id", "release_date", "spotify_track_id", "genres", "label", ) ARTIST_SEARCH_INDEX_PROPS = ( "name", "slugified_name", "country", "spotify_artist_id", "genres", "label", "artwork_url", ) ARTIST_DSP_SEARCH_INDEX_PROPS = { "tiktok": ( "latest_release_date", "avg_duration", "followers", "followers_change_7", "followers_trend_7", "followers_change_28", "followers_trend_28", "followers", "likes", "likes_change_7", "likes_trend_7", "likes_change_28", "likes_trend_28", "videos", "mentions", ), "youtube": ( "latest_release_date", "subscribers", "subscribers_change_7", "subscribers_trend_7", "subscribers_change_28", "subscribers_trend_28", "videos", "views", "views_change_7", "views_trend_7", "views_change_28", "views_trend_28", ), "spotify": ( "latest_release_date", "followers", "followers_change_7", "followers_trend_7", "followers_change_28", "followers_trend_28", "popularity", "listeners", "listeners_change_7", "listeners_trend_7", "listeners_change_28", "listeners_trend_28", "tracks", ), "soundcloud": ( "latest_release_date", "followers", "followers_change_7", "followers_trend_7", "followers_change_28", "followers_trend_28", "plays", "plays_change_7", "plays_trend_7", "plays_change_28", "plays_trend_28", "tracks", "likes", "likes_change_7", "likes_trend_7", "likes_change_28", "likes_trend_28", "comments", "comments_change_7", "comments_trend_7", "comments_change_28", "comments_trend_28", ), } TRACK_SEARCH_INDEX_PROPS = ( "name", "slugified_name", "country", "artist_name", "artwork_url", "cm_artist_id", "spotify_track_id", "genres", "label", ) TRACK_DSP_SEARCH_INDEX_PROPS = { "tiktok": ( "views", "views_change_7", "views_trend_7", "views_change_28", "views_trend_28", "likes", "likes_change_7", "likes_trend_7", "likes_change_28", "likes_trend_28", "comments", "comments_change_7", "comments_trend_7", "comments_change_28", "comments_trend_28", "posts_latest", "posts_latest_change_7", "posts_latest_trend_7", "posts_latest_change_28", "posts_latest_trend_28", "mentions", ), "youtube": ( "latest_release_date", "views", "views_change_7", "views_trend_7", "views_change_28", "views_trend_28", "likes", "likes_change_7", "likes_trend_7", "likes_change_28", "likes_trend_28", "comments", "comments_change_7", "comments_trend_7", "comments_change_28", "comments_trend_28", "videos", ), "spotify": ( "monthly_listeners", "monthly_listeners_change_7", "monthly_listeners_trend_7", "monthly_listeners_change_28", "monthly_listeners_trend_28", "playlists", "playlists_change_7", "playlists_trend_7", "playlists_change_28", "playlists_trend_28", "streams", "streams_change_7", "streams_trend_7", "streams_change_28", "streams_trend_28", "popularity", ), "soundcloud": ( "latest_release_date", "followers", "followers_change_7", "followers_trend_7", "followers_change_28", "followers_trend_28", "plays", "plays_change_7", "plays_trend_7", "plays_change_28", "plays_trend_28", "likes", "likes_change_7", "likes_trend_7", "likes_change_28", "likes_trend_28", "comments", "comments_change_7", "comments_trend_7", "comments_change_28", "comments_trend_28", ), } ARTIST_DSP_TO_INTERPOLATE = ( "soundcloud", "tiktok", "youtube", "spotify", ) TRACK_DSP_TO_INTERPOLATE = ("soundcloud", "spotify") ARTIST_DSP_TO_EXTEND = ( "soundcloud", "tiktok", "youtube", "spotify", ) TRACK_DSP_TO_EXTEND = ("soundcloud", "tiktok", "spotify", "youtube") ARTIST_TRENDS_AND_CHANGES_CHARTS = { "spotify": ( "listeners", "followers", ), "tiktok": ( "followers", "likes", ), "youtube": ( "subscribers", "views", ), "soundcloud": ( "followers", "plays", "likes", "comments", ), } TRACK_TRENDS_AND_CHANGES_CHARTS = { "spotify": ( "streams", "monthly_listeners", ), "tiktok": ("views", "likes", "comments", "posts_latest"), "youtube": ( "views", "likes", "comments", ), "soundcloud": ( "plays", "likes", "comments", "followers", ), } NO_IMPACT_RELEASE_DATE_TRACK = {"spotify": ("monthly_listeners",), "soundcloud": ("followers",)} # S3 keys BASE_PATH = PurePosixPath("search") # ------------ Artist section ------------ ARTIST_BASE_PATH = BASE_PATH / "artist" ARTIST_RAW_DATA = ARTIST_BASE_PATH / "raw_data" ARTIST_SPOTIFY_RAW_DATA = ARTIST_RAW_DATA / "spotify" ARTIST_SOUNDCLOUD_RAW_DATA = ARTIST_RAW_DATA / "soundcloud" ARTIST_YOUTUBE_RAW_DATA = ARTIST_RAW_DATA / "youtube" ARTIST_TIKTOK_RAW_DATA = ARTIST_RAW_DATA / "tiktok" ARTIST_SEARCH_INDEX_DATA_PATH = ARTIST_BASE_PATH / "index_data" ARTIST_SEARCH_DATA_PATH = ARTIST_BASE_PATH / "agg_data" # ------------ Track section ------------ TRACK_BASE_PATH = BASE_PATH / "track" TRACK_RAW_DATA = TRACK_BASE_PATH / "raw_data" TRACK_SPOTIFY_RAW_DATA = TRACK_RAW_DATA / "spotify" TRACK_SOUNDCLOUD_RAW_DATA = TRACK_RAW_DATA / "soundcloud" TRACK_YOUTUBE_RAW_DATA = TRACK_RAW_DATA / "youtube" TRACK_TIKTOK_RAW_DATA = TRACK_RAW_DATA / "tiktok" TRACK_SEARCH_INDEX_DATA_PATH = TRACK_BASE_PATH / "index_data" TRACK_SEARCH_DATA_PATH = TRACK_BASE_PATH / "agg_data" DEFAULT_ES_INDEX_NULL = -99999 ARTIST_SEARCH_INDEX_MAPPING = { "mappings": { "properties": { "soundcloud": { "properties": { "latest_release_date": {"type": "date", "null_value": "1970-01-01"}, "followers": {"type": "long", "null_value": DEFAULT_ES_INDEX_NULL}, "followers_change_7": {"type": "long", "null_value": DEFAULT_ES_INDEX_NULL}, "followers_trend_7": {"type": "float", "null_value": DEFAULT_ES_INDEX_NULL}, "followers_change_28": {"type": "long", "null_value": DEFAULT_ES_INDEX_NULL}, "followers_trend_28": {"type": "float", "null_value": DEFAULT_ES_INDEX_NULL}, "plays": {"type": "long", "null_value": DEFAULT_ES_INDEX_NULL}, "plays_change_7": {"type": "long", "null_value": DEFAULT_ES_INDEX_NULL}, "plays_trend_7": {"type": "float", "null_value": DEFAULT_ES_INDEX_NULL}, "plays_change_28": {"type": "long", "null_value": DEFAULT_ES_INDEX_NULL}, "plays_trend_28": {"type": "float", "null_value": DEFAULT_ES_INDEX_NULL}, "likes": {"type": "long", "null_value": DEFAULT_ES_INDEX_NULL}, "likes_change_7": {"type": "long", "null_value": DEFAULT_ES_INDEX_NULL}, "likes_trend_7": {"type": "float", "null_value": DEFAULT_ES_INDEX_NULL}, "likes_change_28": {"type": "long", "null_value": DEFAULT_ES_INDEX_NULL}, "likes_trend_28": {"type": "float", "null_value": DEFAULT_ES_INDEX_NULL}, "tracks": {"type": "long", "null_value": DEFAULT_ES_INDEX_NULL}, "comments": {"type": "long", "null_value": DEFAULT_ES_INDEX_NULL}, "comments_change_7": {"type": "long", "null_value": DEFAULT_ES_INDEX_NULL}, "comments_trend_7": {"type": "float", "null_value": DEFAULT_ES_INDEX_NULL}, "comments_change_28": {"type": "long", "null_value": DEFAULT_ES_INDEX_NULL}, "comments_trend_28": {"type": "float", "null_value": DEFAULT_ES_INDEX_NULL}, } }, "tiktok": { "properties": { "followers": {"type": "long", "null_value": DEFAULT_ES_INDEX_NULL}, "followers_change_7": {"type": "long", "null_value": DEFAULT_ES_INDEX_NULL}, "followers_trend_7": {"type": "float", "null_value": DEFAULT_ES_INDEX_NULL}, "followers_change_28": {"type": "long", "null_value": DEFAULT_ES_INDEX_NULL}, "followers_trend_28": {"type": "float", "null_value": DEFAULT_ES_INDEX_NULL}, "avg_duration": {"type": "long", "null_value": DEFAULT_ES_INDEX_NULL}, "likes": {"type": "long", "null_value": DEFAULT_ES_INDEX_NULL}, "likes_change_7": {"type": "long", "null_value": DEFAULT_ES_INDEX_NULL}, "likes_trend_7": {"type": "float", "null_value": DEFAULT_ES_INDEX_NULL}, "likes_change_28": {"type": "long", "null_value": DEFAULT_ES_INDEX_NULL}, "likes_trend_28": {"type": "float", "null_value": DEFAULT_ES_INDEX_NULL}, "mentions": {"type": "long", "null_value": DEFAULT_ES_INDEX_NULL}, "videos": {"type": "long", "null_value": DEFAULT_ES_INDEX_NULL}, "latest_release_date": {"type": "date", "null_value": "1970-01-01"}, } }, "youtube": { "properties": { "latest_release_date": {"type": "date", "null_value": "1970-01-01"}, "subscribers": {"type": "long", "null_value": DEFAULT_ES_INDEX_NULL}, "subscribers_change_7": {"type": "long", "null_value": DEFAULT_ES_INDEX_NULL}, "subscribers_trend_7": {"type": "float", "null_value": DEFAULT_ES_INDEX_NULL}, "subscribers_change_28": {"type": "long", "null_value": DEFAULT_ES_INDEX_NULL}, "subscribers_trend_28": {"type": "float", "null_value": DEFAULT_ES_INDEX_NULL}, "videos": {"type": "long", "null_value": DEFAULT_ES_INDEX_NULL}, "views": {"type": "long", "null_value": DEFAULT_ES_INDEX_NULL}, "views_change_7": {"type": "long", "null_value": DEFAULT_ES_INDEX_NULL}, "views_trend_7": {"type": "float", "null_value": DEFAULT_ES_INDEX_NULL}, "views_change_28": {"type": "long", "null_value": DEFAULT_ES_INDEX_NULL}, "views_trend_28": {"type": "float", "null_value": DEFAULT_ES_INDEX_NULL}, } }, "spotify": { "properties": { "latest_release_date": {"type": "date", "null_value": "1970-01-01"}, "listeners": {"type": "long", "null_value": DEFAULT_ES_INDEX_NULL}, "listeners_change_7": {"type": "long", "null_value": DEFAULT_ES_INDEX_NULL}, "listeners_trend_7": {"type": "float", "null_value": DEFAULT_ES_INDEX_NULL}, "listeners_change_28": {"type": "long", "null_value": DEFAULT_ES_INDEX_NULL}, "listeners_trend_28": {"type": "float", "null_value": DEFAULT_ES_INDEX_NULL}, "followers": {"type": "long", "null_value": DEFAULT_ES_INDEX_NULL}, "followers_change_7": {"type": "long", "null_value": DEFAULT_ES_INDEX_NULL}, "followers_trend_7": {"type": "float", "null_value": DEFAULT_ES_INDEX_NULL}, "followers_change_28": {"type": "long", "null_value": DEFAULT_ES_INDEX_NULL}, "followers_trend_28": {"type": "float", "null_value": DEFAULT_ES_INDEX_NULL}, "popularity": {"type": "long", "null_value": DEFAULT_ES_INDEX_NULL}, "tracks": {"type": "long", "null_value": DEFAULT_ES_INDEX_NULL}, } }, } } } TRACK_SEARCH_INDEX_MAPPING = { "mappings": { "properties": { "soundcloud": { "properties": { "latest_release_date": {"type": "date", "null_value": "1970-01-01"}, "followers": {"type": "long", "null_value": DEFAULT_ES_INDEX_NULL}, "followers_change_7": {"type": "long", "null_value": DEFAULT_ES_INDEX_NULL}, "followers_trend_7": {"type": "float", "null_value": DEFAULT_ES_INDEX_NULL}, "followers_change_28": {"type": "long", "null_value": DEFAULT_ES_INDEX_NULL}, "followers_trend_28": {"type": "float", "null_value": DEFAULT_ES_INDEX_NULL}, "plays": {"type": "long", "null_value": DEFAULT_ES_INDEX_NULL}, "plays_change_7": {"type": "long", "null_value": DEFAULT_ES_INDEX_NULL}, "plays_trend_7": {"type": "float", "null_value": DEFAULT_ES_INDEX_NULL}, "plays_change_28": {"type": "long", "null_value": DEFAULT_ES_INDEX_NULL}, "plays_trend_28": {"type": "float", "null_value": DEFAULT_ES_INDEX_NULL}, "likes": {"type": "long", "null_value": DEFAULT_ES_INDEX_NULL}, "likes_change_7": {"type": "long", "null_value": DEFAULT_ES_INDEX_NULL}, "likes_trend_7": {"type": "float", "null_value": DEFAULT_ES_INDEX_NULL}, "likes_change_28": {"type": "long", "null_value": DEFAULT_ES_INDEX_NULL}, "likes_trend_28": {"type": "float", "null_value": DEFAULT_ES_INDEX_NULL}, "comments": {"type": "long", "null_value": DEFAULT_ES_INDEX_NULL}, "comments_change_7": {"type": "long", "null_value": DEFAULT_ES_INDEX_NULL}, "comments_trend_7": {"type": "float", "null_value": DEFAULT_ES_INDEX_NULL}, "comments_change_28": {"type": "long", "null_value": DEFAULT_ES_INDEX_NULL}, "comments_trend_28": {"type": "float", "null_value": DEFAULT_ES_INDEX_NULL}, } }, "tiktok": { "properties": { "views": {"type": "long", "null_value": DEFAULT_ES_INDEX_NULL}, "views_change_7": {"type": "long", "null_value": DEFAULT_ES_INDEX_NULL}, "views_trend_7": {"type": "float", "null_value": DEFAULT_ES_INDEX_NULL}, "views_change_28": {"type": "long", "null_value": DEFAULT_ES_INDEX_NULL}, "views_trend_28": {"type": "float", "null_value": DEFAULT_ES_INDEX_NULL}, "posts_latest": {"type": "long", "null_value": DEFAULT_ES_INDEX_NULL}, "posts_latest_change_7": {"type": "long", "null_value": DEFAULT_ES_INDEX_NULL}, "posts_latest_trend_7": {"type": "float", "null_value": DEFAULT_ES_INDEX_NULL}, "posts_latest_change_28": {"type": "long", "null_value": DEFAULT_ES_INDEX_NULL}, "posts_latest_trend_28": {"type": "float", "null_value": DEFAULT_ES_INDEX_NULL}, "mentions": {"type": "long", "null_value": DEFAULT_ES_INDEX_NULL}, "likes": {"type": "long", "null_value": DEFAULT_ES_INDEX_NULL}, "likes_change_7": {"type": "long", "null_value": DEFAULT_ES_INDEX_NULL}, "likes_trend_7": {"type": "float", "null_value": DEFAULT_ES_INDEX_NULL}, "likes_change_28": {"type": "long", "null_value": DEFAULT_ES_INDEX_NULL}, "likes_trend_28": {"type": "float", "null_value": DEFAULT_ES_INDEX_NULL}, "comments": {"type": "long", "null_value": DEFAULT_ES_INDEX_NULL}, "comments_change_7": {"type": "long", "null_value": DEFAULT_ES_INDEX_NULL}, "comments_trend_7": {"type": "float", "null_value": DEFAULT_ES_INDEX_NULL}, "comments_change_28": {"type": "long", "null_value": DEFAULT_ES_INDEX_NULL}, "comments_trend_28": {"type": "float", "null_value": DEFAULT_ES_INDEX_NULL}, } }, "youtube": { "properties": { "latest_release_date": {"type": "date", "null_value": "1970-01-01"}, "videos": {"type": "long", "null_value": DEFAULT_ES_INDEX_NULL}, "views": {"type": "long", "null_value": DEFAULT_ES_INDEX_NULL}, "views_change_7": {"type": "long", "null_value": DEFAULT_ES_INDEX_NULL}, "views_trend_7": {"type": "float", "null_value": DEFAULT_ES_INDEX_NULL}, "views_change_28": {"type": "long", "null_value": DEFAULT_ES_INDEX_NULL}, "views_trend_28": {"type": "float", "null_value": DEFAULT_ES_INDEX_NULL}, "likes": {"type": "long", "null_value": DEFAULT_ES_INDEX_NULL}, "likes_change_7": {"type": "long", "null_value": DEFAULT_ES_INDEX_NULL}, "likes_trend_7": {"type": "float", "null_value": DEFAULT_ES_INDEX_NULL}, "likes_change_28": {"type": "long", "null_value": DEFAULT_ES_INDEX_NULL}, "likes_trend_28": {"type": "float", "null_value": DEFAULT_ES_INDEX_NULL}, "comments": {"type": "long", "null_value": DEFAULT_ES_INDEX_NULL}, "comments_change_7": {"type": "long", "null_value": DEFAULT_ES_INDEX_NULL}, "comments_trend_7": {"type": "float", "null_value": DEFAULT_ES_INDEX_NULL}, "comments_change_28": {"type": "long", "null_value": DEFAULT_ES_INDEX_NULL}, "comments_trend_28": {"type": "float", "null_value": DEFAULT_ES_INDEX_NULL}, } }, "spotify": { "properties": { "streams": {"type": "long", "null_value": DEFAULT_ES_INDEX_NULL}, "streams_change_7": {"type": "long", "null_value": DEFAULT_ES_INDEX_NULL}, "streams_trend_7": {"type": "float", "null_value": DEFAULT_ES_INDEX_NULL}, "streams_change_28": {"type": "long", "null_value": DEFAULT_ES_INDEX_NULL}, "streams_trend_28": {"type": "float", "null_value": DEFAULT_ES_INDEX_NULL}, "popularity": {"type": "long", "null_value": DEFAULT_ES_INDEX_NULL}, "playlists": {"type": "long", "null_value": DEFAULT_ES_INDEX_NULL}, "playlists_change_7": {"type": "long", "null_value": DEFAULT_ES_INDEX_NULL}, "playlists_trend_7": {"type": "float", "null_value": DEFAULT_ES_INDEX_NULL}, "playlists_change_28": {"type": "long", "null_value": DEFAULT_ES_INDEX_NULL}, "playlists_trend_28": {"type": "float", "null_value": DEFAULT_ES_INDEX_NULL}, "monthly_listeners": {"type": "long", "null_value": DEFAULT_ES_INDEX_NULL}, "monthly_listeners_change_7": {"type": "long", "null_value": DEFAULT_ES_INDEX_NULL}, "monthly_listeners_trend_7": {"type": "float", "null_value": DEFAULT_ES_INDEX_NULL}, "monthly_listeners_change_28": {"type": "long", "null_value": DEFAULT_ES_INDEX_NULL}, "monthly_listeners_trend_28": {"type": "float", "null_value": DEFAULT_ES_INDEX_NULL}, } }, } } }