using System; using System.Collections.Concurrent; using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; using MoreLinq; using PetaPoco.Business; using Sony.Filtr.Contracts.Abstractions; using Sony.Filtr.Contracts.Definitions; using Sony.Filtr.Contracts.Entities; using Sony.Filtr.Core.EditorialPlaylists; using Sony.Filtr.Core.SpotifyBrowse.Data; using Sony.Filtr.Core.SpotifyRegion; using Sony.Filtr.DistributedCaching; using Sony.Filtr.SpotifyWebAPI.Model; using Sony.Filtr.Utility.Extensions; namespace Sony.Filtr.Core.SpotifyBrowse { public class SpotifyBrowseManager { private readonly DistributedCacheHandler _cacheHandler; private readonly SpotifyBrowseFactory _spotifyBrowseFactory; private readonly EditorialPlaylistManager _editorialPlaylistManager; private readonly EditorialPlaylistStatsManager _editorialPlaylistStatsManager; private readonly IApplicationInstanceManager _applicationInstanceManager; private readonly SpotifyRegionManager _spotifyRegionManager; private readonly IBuzzAccountManager _buzzAccountManager; private readonly string _browseDatesCacheKey = "spotifyBrowseDates-2"; private readonly string _browseUtcDatesCacheKey = "spotifyBrowseUtcDates-2"; private readonly string _browseDataCacheKey = "spotifyBrowseData-2"; private readonly string _browseCategoriesCacheKey = "spotifyBrowseCategories-2"; private readonly string _browseMarketsCacheKey = "spotifyBrowseMarkets-2"; private readonly int _firstBracketLimit = 10; private readonly int _secondBracketLimit = 20; public SpotifyBrowseManager(SpotifyBrowseFactory spotifyBrowseFactory, DistributedCacheHandler distributedCacheHandler, EditorialPlaylistManager editorialPlaylistManager, EditorialPlaylistStatsManager editorialPlaylistStatsManager, IApplicationInstanceManager applicationInstanceManager, SpotifyRegionManager spotifyRegionManager, IBuzzAccountManager buzzAccountManager) { _spotifyBrowseFactory = spotifyBrowseFactory; _cacheHandler = distributedCacheHandler; _editorialPlaylistManager = editorialPlaylistManager; _editorialPlaylistStatsManager = editorialPlaylistStatsManager; _applicationInstanceManager = applicationInstanceManager; _spotifyRegionManager = spotifyRegionManager; _buzzAccountManager = buzzAccountManager; } public List GetSpotifyBrowseData(DateTime date) { return PetaPocoRepository.ReadOnlyInstance.Fetch("WHERE Date=@0", date); } private async Task> GetSpotifyBrowseChartsAsync(DateTime date, string region, string categoryId = null) { var cacheKey = GetBrowseDataCacheKey(region, date, categoryId); var browseData = _cacheHandler.Get(cacheKey) as List; if (browseData == null) { browseData = await _spotifyBrowseFactory.GetSpotifyBrowseCategoryStatsAsync(region, date); _cacheHandler.Put(cacheKey, browseData); } return browseData; } public async Task> GetSpotifyBrowseDatesAsync(string region = null, bool ignoreCache = false) { var cacheKey = GetDateCacheKey(region); var dates = _cacheHandler.Get(cacheKey) as List; if (dates == null || ignoreCache) { dates = await _spotifyBrowseFactory.GetSpotifyBrowseDatesAsync(region); _cacheHandler.Put(cacheKey, dates); } return dates; } public async Task> GetSpotifyBrowseUtcDatesAsync(string region = null, bool ignoreCache = false) { var cacheKey = GetUtcDateCacheKey(region); var utcDates = _cacheHandler.Get(cacheKey) as Dictionary; if (utcDates == null || ignoreCache) { utcDates = await _spotifyBrowseFactory.GetSpotifyBrowseUtcDatesAsync(region); _cacheHandler.Put(cacheKey, utcDates); } return utcDates; } public async Task> GetSpotifyBrowseMarketsAsync() { var cacheKey = _browseMarketsCacheKey; var markets = _cacheHandler.Get(cacheKey) as List; if (markets == null) { markets = await _spotifyBrowseFactory.GetSpotifyBrowseMarketsAsync(); _cacheHandler.Put(cacheKey, markets); } return markets; } public async Task> GetSpotifyBrowseCategoriesAsync(string region) { if (region == null) return new List(); var cacheKey = GetCategoriesCacheKey(region); var categories = _cacheHandler.Get(cacheKey) as List; if (categories == null) { categories = await _spotifyBrowseFactory.GetSpotifyBrowseCategoriesAsync(region); _cacheHandler.Put(cacheKey, categories); } return categories; } public async Task GetSpotifyBrowseCategoryStatsAsync(DateTime date, Application application) { var browseItems = await GetSpotifyBrowseChartsAsync(date, application.SpotifyRegionCode); var categoryStats = await CalculateCategoryStatsAsync(browseItems, application); var categoryStatsList = categoryStats.ToList(); var totalStats = new SpotifyBrowseCategoryOverallStats(); totalStats.Date = date; totalStats.TotalPlaylistCount = categoryStatsList.Sum(kvp => kvp.Value.PlaylistCount); totalStats.TotalSonyPlaylists = new SonyAmounts() { SonyPlaylists = categoryStatsList.Sum(kvp => kvp.Value.SonyPlaylists.SonyPlaylists), LocalSonyPlaylists = categoryStatsList.Sum(kvp => kvp.Value.SonyPlaylists.LocalSonyPlaylists), }; totalStats.TotalSonyPlaylistsInFirstBracket = new SonyAmounts() { SonyPlaylists = categoryStatsList.Sum(kvp => kvp.Value.SonyPlaylistsInFirstBracket.SonyPlaylists), LocalSonyPlaylists = categoryStatsList.Sum(kvp => kvp.Value.SonyPlaylistsInFirstBracket.LocalSonyPlaylists), }; totalStats.TotalSonyPlaylistsInSecondBracket = new SonyAmounts() { SonyPlaylists = categoryStatsList.Sum(kvp => kvp.Value.SonyPlaylistsInSecondBracket.SonyPlaylists), LocalSonyPlaylists = categoryStatsList.Sum(kvp => kvp.Value.SonyPlaylistsInSecondBracket.LocalSonyPlaylists), }; totalStats.TotalSonyPlaylistsAfterBrackets = new SonyAmounts() { SonyPlaylists = categoryStatsList.Sum(kvp => kvp.Value.SonyPlaylistsAfterBrackets.SonyPlaylists), LocalSonyPlaylists = categoryStatsList.Sum(kvp => kvp.Value.SonyPlaylistsAfterBrackets.LocalSonyPlaylists), }; totalStats.CategoryStats = categoryStats; totalStats.OwnerSpread = browseItems.GroupBy(b => b.BuzzCategoryId).Select(b => new OwnerAmount() { BuzzCategoryId = b.Key, BuzzCategoryName = b.First().BuzzCategoryName, Amount = b.Count() }).OrderByDescending(p=> p.Amount).ToList(); //foreach (var item in browseItems) //{ // if (totalStats.OwnerSpread.Any(o => o.BuzzCategoryId == item.BuzzCategoryId)) // { // totalStats.OwnerSpread.First(o => o.BuzzCategoryId == item.BuzzCategoryId).Amount++; // } // else // { // totalStats.OwnerSpread.Add(new OwnerAmount() { Amount = 1, BuzzCategoryId = item.BuzzCategoryId, BuzzCategoryName = item.BuzzCategoryName }); // } //} return totalStats; } private async Task> CalculateCategoryStatsAsync(List browseItems, Application application) { var browseStats = new Dictionary(); var sonyBuzzCategoryId = (int)StaticBuzzCategory.SonyMusic; var buzzUsers = await _buzzAccountManager.GetBuzzUsersAsync(buzzCategoryId: sonyBuzzCategoryId, musicServiceId: (int)MusicService.Spotify); var buzzUserDic = buzzUsers.ToDictionary(k => k.Username, v=> v, StringComparer.InvariantCultureIgnoreCase); foreach (var browseItem in browseItems) { if (!browseStats.ContainsKey(browseItem.CategoryId)) { browseStats.Add(browseItem.CategoryId, new SpotifyBrowseCategoryStat()); browseStats[browseItem.CategoryId].SonyPlaylists = new SonyAmounts(); browseStats[browseItem.CategoryId].SonyPlaylistsInFirstBracket = new SonyAmounts(); browseStats[browseItem.CategoryId].SonyPlaylistsInSecondBracket = new SonyAmounts(); browseStats[browseItem.CategoryId].SonyPlaylistsAfterBrackets = new SonyAmounts(); } browseStats[browseItem.CategoryId].CategoryName = browseItem.CategoryName; browseStats[browseItem.CategoryId].CategoryId = browseItem.CategoryId; browseStats[browseItem.CategoryId].PlaylistCount++; // More stats! var buzzUser = buzzUserDic.GetValueOrDefault(browseItem.OwnerUsername); if (buzzUser != null) { browseStats[browseItem.CategoryId].SonyPlaylists.SonyPlaylists++; var isLocalList = buzzUser.CountryCode == application.SpotifyRegionCode; if (isLocalList) browseStats[browseItem.CategoryId].SonyPlaylists.LocalSonyPlaylists++; if (browseItem.Position <= _firstBracketLimit) { browseStats[browseItem.CategoryId].SonyPlaylistsInFirstBracket.SonyPlaylists++; if (isLocalList) browseStats[browseItem.CategoryId].SonyPlaylistsInFirstBracket.LocalSonyPlaylists++; } else if (browseItem.Position <= _secondBracketLimit) { browseStats[browseItem.CategoryId].SonyPlaylistsInSecondBracket.SonyPlaylists++; if (isLocalList) browseStats[browseItem.CategoryId].SonyPlaylistsInSecondBracket.LocalSonyPlaylists++; } else { browseStats[browseItem.CategoryId].SonyPlaylistsAfterBrackets.SonyPlaylists++; if (isLocalList) browseStats[browseItem.CategoryId].SonyPlaylistsAfterBrackets.LocalSonyPlaylists++; } } } return browseStats; } public async Task GetSpotifyBrowseCategoryTopListAsync(Application application, string categoryId, DateTime date) { var browseTopList = new SpotifyBrowseTopList() { CategoryId = categoryId, ApplicationId = application.ID, SonyPlaylists = new SonyAmounts(), SonyPlaylistsInFirstBracket = new SonyAmounts(), SonyPlaylistsInSecondBracket = new SonyAmounts(), SonyPlaylistsAfterBrackets = new SonyAmounts(), OwnerSpread = new List() }; string region = application.SpotifyRegionCode; var categories = await GetSpotifyBrowseCategoriesAsync(region); var browseCategory = categories.FirstOrDefault(c => c.CategoryId.Equals(categoryId, StringComparison.InvariantCultureIgnoreCase)); browseTopList.CategoryName = browseCategory?.CategoryName ?? string.Empty; browseTopList.Items = await _spotifyBrowseFactory.GetSpotifyBrowseTopListAsync(region, categoryId, date); var statDate = date; var playlistIds = browseTopList.Items.Select(t => new SpotifyLink(t.PlaylistURI).ExtractPlaylistID()).ToList(); var playlistStats = await _editorialPlaylistStatsManager.GetPlaylistStatsAsync(playlistIds, statDate, statDate); foreach (var chartItem in browseTopList.Items) { var isLocal = chartItem.PlaylistMarket.Equals(region, StringComparison.InvariantCultureIgnoreCase); if (chartItem.IsSony) { browseTopList.SonyPlaylists.SonyPlaylists++; if (isLocal) browseTopList.SonyPlaylists.LocalSonyPlaylists++; if (chartItem.Position <= _firstBracketLimit) { browseTopList.SonyPlaylistsInFirstBracket.SonyPlaylists++; if (isLocal) browseTopList.SonyPlaylistsInFirstBracket.LocalSonyPlaylists++; } else if (chartItem.Position <= _secondBracketLimit) { browseTopList.SonyPlaylistsInSecondBracket.SonyPlaylists++; if (isLocal) browseTopList.SonyPlaylistsInSecondBracket.LocalSonyPlaylists++; } else { browseTopList.SonyPlaylistsAfterBrackets.SonyPlaylists++; if (isLocal) browseTopList.SonyPlaylistsAfterBrackets.LocalSonyPlaylists++; } } var followers = (int?)playlistStats.FirstOrDefault(p => p.PlaylistId == new SpotifyLink(chartItem.PlaylistURI).ExtractPlaylistID() && p.Date.Day == statDate.Day)?.Subscribers; chartItem.Followers = followers; if (!string.IsNullOrWhiteSpace(chartItem.BrowseMarketString)) { chartItem.AllBrowseMarkets = chartItem.BrowseMarketString.Split(','); } if (browseTopList.OwnerSpread.Any(o => o.BuzzCategoryId == chartItem.BuzzCategoryId)) { browseTopList.OwnerSpread.First(o => o.BuzzCategoryId == chartItem.BuzzCategoryId).Amount++; } else { browseTopList.OwnerSpread.Add(new OwnerAmount() { Amount = 1, BuzzCategoryId = chartItem.BuzzCategoryId, BuzzCategoryName = chartItem.BuzzCategoryName }); } } browseTopList.OwnerSpread = browseTopList.OwnerSpread.OrderByDescending(o => o.Amount).ToList(); return browseTopList; } public async Task> GetSpotifyBrowseForExportAsync(string market, DateTime date) { return await _spotifyBrowseFactory.GetSpotifyBrowseForExportAsync(market, date); } private string GetDateCacheKey(string region) { if (string.IsNullOrWhiteSpace(region)) return _browseDatesCacheKey + "_all"; return _browseDatesCacheKey + "_" + region.ToUpper(); } private string GetUtcDateCacheKey(string region) { if (string.IsNullOrWhiteSpace(region)) return _browseUtcDatesCacheKey + "_all"; return _browseUtcDatesCacheKey + "_" + region.ToUpper(); } private string GetCategoriesCacheKey(string region) { return _browseCategoriesCacheKey + "_1_" + region.ToUpper(); } private string GetBrowseDataCacheKey(string region, DateTime date, string categoryId) { return _browseDataCacheKey + "_" + region.ToUpper() + "_" + date.ToString("yyyy-MM-dd") + "_" + categoryId; } public async Task ClearCachesAsync() { var dates = await GetSpotifyBrowseDatesAsync(null, ignoreCache: true); var applications = _applicationInstanceManager.GetApplications(); var applicationRegions = applications.Select(a => a.SpotifyRegionCode).Distinct().ToList(); var spotifyRegions = await _spotifyRegionManager.GetAvailableRegionsAsync(); var allRegions = applicationRegions.Union(spotifyRegions).Distinct(StringComparer.InvariantCultureIgnoreCase); _cacheHandler.Remove(GetDateCacheKey(null)); _cacheHandler.Remove(GetUtcDateCacheKey(null)); foreach (var region in allRegions) { _cacheHandler.Remove(GetDateCacheKey(region)); _cacheHandler.Remove(GetUtcDateCacheKey(region)); _cacheHandler.Remove(GetCategoriesCacheKey(region)); foreach (var date in dates) { _cacheHandler.Remove(GetBrowseDataCacheKey(region, date, null)); } } } private void ClearCaches(string region, DateTime date) { _cacheHandler.Remove(GetDateCacheKey(region)); _cacheHandler.Remove(GetDateCacheKey(null)); _cacheHandler.Remove(GetBrowseDataCacheKey(region, date, null)); _cacheHandler.Remove(GetCategoriesCacheKey(region)); _cacheHandler.Remove(GetUtcDateCacheKey(region)); _cacheHandler.Remove(GetUtcDateCacheKey(null)); } private void ClearCategoryCache(string region, DateTime date, string category) { _cacheHandler.Remove(GetBrowseDataCacheKey(region, date, category)); } public async Task SaveBrowseDataAsync(IDictionary> playlistsPerCategory, DateTime today, string region) { List spotifyBrowseItems = playlistsPerCategory.SelectMany(c => c.Value.Index().Select(p => new SpotifyBrowseItem2() { Category = c.Key, Date = today, Market = region, PlaylistId = p.Value.id, PlaylistName = p.Value.name, Position = p.Key + 1 })).ToList(); if (spotifyBrowseItems.Any()) { PetaPocoRepository.Instance.Db.BulkInsert(spotifyBrowseItems); await _spotifyBrowseFactory.SetSpotifyBrowseDateAsync(region, today); } ClearCaches(region, today); spotifyBrowseItems.Select(i => i.Category).ForEach(c => ClearCategoryCache(region, today, c)); } public async Task> GetSpotifyBrowsePlaylistsAsync(string playlistUri, DateTime startDate, DateTime endDate) { return await _spotifyBrowseFactory.GetSpotifyBrowsePlaylistsSimpleItemAsync(playlistUri, startDate, endDate); } public async Task> GetSpotifyBrowsePlaylistsSummaryAsync(DateTime date, List playlistIds) { return await _spotifyBrowseFactory.GetSpotifyBrowsePlaylistsSummaryAsync(playlistIds, date, date); } public string GetSpotifyBrowseExportFileName(DateTime browseDate) { return $"SpotifyBrowse-global-{browseDate.ToString("yyyy-MM-dd")}.xlsx"; } public string SpotifyBrowseS3BucketName { get { return "filtr-spotify-browse-export"; } } } }