using System; using System.Collections.Concurrent; using System.Collections.Generic; using System.Linq; using System.Threading; using System.Threading.Tasks; using MoreLinq; using NLog; using Sony.Filtr.ApolloAPI; using Sony.Filtr.ApolloAPI.Models.Apple; using Sony.Filtr.AppleMusic; using Sony.Filtr.AppleMusic.Curators; using Sony.Filtr.AppleMusic.Data; using Sony.Filtr.AppleMusic.Data.Internal; using Sony.Filtr.AppleMusic.Playlists; using Sony.Filtr.Buzz; using Sony.Filtr.Contracts.Definitions; using Sony.Filtr.Contracts.Entities; using Sony.Filtr.Contracts.Entities.Buzz; using Sony.Filtr.Functional; using Sony.Filtr.Tasks.Helpers; using Sony.Filtr.Utility; using Sony.Filtr.Utility.ConsoleUtility; using Sony.Filtr.Utility.Extensions; namespace Sony.Filtr.Tasks.Tasks.AppleMusic { public class UpdateAppleMusicPlaylistsMostStreamedTask : IScheduledTask { private readonly IApolloAppleWebApi _appleMusicApi; private readonly AppleMusicPlaylistManager _appleMusicPlaylistManager; private readonly AppleMusicCuratorManager _appleMusicCuratorManager; private readonly BuzzAccountManager _buzzAccountManager; private readonly Logger _logger; private int _appleCuratedPlaylistsProcess; private readonly List _majorStorefronts = new List { "us", "gb", "au", "ca", "de", "fr" }; private HashSet> _historicTracklistLookup; public UpdateAppleMusicPlaylistsMostStreamedTask(IApolloAppleWebApi appleMusicApi, AppleMusicPlaylistManager appleMusicPlaylistManager, AppleMusicCuratorManager appleMusicCuratorManager, BuzzAccountManager buzzAccountManager) { _appleMusicApi = appleMusicApi; _appleMusicPlaylistManager = appleMusicPlaylistManager; _appleMusicCuratorManager = appleMusicCuratorManager; _buzzAccountManager = buzzAccountManager; _logger = LogManager.GetLogger("UpdateAppleMusicPlaylistsMostStreamed"); } private static int GetApplePlaylistsBatchSize() { return Maybe.GetAppSettingsIntOrDefault("UpdateAppleMusicPlaylistsMostStreamedTask_Apple_Playlist_Batch_Size", Constants.Apple.ApiPlaylistBatchSize); } public async Task ExecuteAsync(Guid scheduledTaskLogId) { var currentUtcDate = DateTime.UtcNow.Date; var whitelistedPlaylists = _appleMusicPlaylistManager.GetAppleMusicWhitelistedCuratedPlaylists(); // Set basic values for playlists that have not been imported yet (to handle apple curators correctly) await ImportNewPlaylistsAsync(); _logger.Debug($"Begin update"); PaginatedContent paginatedPlaylists = new PaginatedContent(); foreach (var storefront in _majorStorefronts) { var storeFrontSpecific = await _appleMusicPlaylistManager.GetPlaylistsMostStreamedAsync(storefront, 1000); foreach (var playlist in storeFrontSpecific.Items) { paginatedPlaylists.Items.Add(new AppleMusicPlaylistWithStorefront() { Playlist = playlist, Storefront = storefront }); } } var allPlaylists = paginatedPlaylists.Items.ToList(); _logger.Debug($"{allPlaylists.Count} playlists in total."); var historicTracklistDates = _appleMusicPlaylistManager.GetHistoricTrackListsForDateAsync(currentUtcDate); _historicTracklistLookup = historicTracklistDates.Select(p => new Tuple(p.Storefront.ToLowerInvariant(), p.PlaylistId)).ToHashSet(); _logger.Debug($"{historicTracklistDates.Count} existing historic tracklists for current date."); Dictionary appleMusicBuzzAccountsById = new Dictionary(); try { appleMusicBuzzAccountsById = await GetAndCreateAppleBuzzAccountsAsync(allPlaylists.Select(x=>x.Playlist).ToList()); } catch (Exception ex) { _logger.Error(ex, "Error in adding/getting buzz users."); } //For Apple curated lists: get and save tracklist for all major storefronts. _logger.Debug($"Updating {allPlaylists.Count} Apple curated playlists."); Dictionary playlistMasterStorefront = allPlaylists.Select(y => y.Playlist).GroupBy(pl => pl.Id).Select(g => g.First()).ToDictionary(k => k.Id, v => GetStorefront(v, appleMusicBuzzAccountsById)); await UpdateApplePlaylistsAsync(allPlaylists, _majorStorefronts, playlistMasterStorefront, currentUtcDate, whitelistedPlaylists); _logger.Debug($"Done updating playlists"); _logger.Debug("So over this."); return null; } private async Task ImportNewPlaylistsAsync() { var newPlaylists = await _appleMusicPlaylistManager.GetNotUpdatedPlaylistsAsync(); _logger.Debug($"Doing first import of {newPlaylists.Count} playlists."); await newPlaylists.Batch(GetApplePlaylistsBatchSize()).ForEachAsync(5, async playlistBatch => { try { await UpdatePlaylistsInfoAsync(AppleMusicPlaylistManager.DefaultStorefrontForTrackList, playlistBatch.ToList()); } catch (Exception ex) { _logger.Error(ex, "Error while importing playlists"); } }); } private async Task> GetAndCreateAppleBuzzAccountsAsync(List allPlaylists) { var appleBuzzAccountsById = new ConcurrentDictionary((await _buzzAccountManager.GetBuzzUsersAsync(musicServiceId: (int)MusicService.AppleMusic)).ToDictionary(a => a.Username, a => a)); var allCuratorIdsWithType = allPlaylists.Where(p => p.CuratorId.HasValue).DistinctBy(p => p.CuratorId.Value).Select(p => new CuratorIdWithType { CuratorId = p.CuratorId.Value, CuratorType = p.CuratorType }).ToList(); var missingAccountsWithType = allCuratorIdsWithType.Where(c => !appleBuzzAccountsById.ContainsKey(c.CuratorId.ToString())).ToList(); _logger.Debug($"Adding {missingAccountsWithType.Count()} Apple Music buzz accounts."); await missingAccountsWithType.ForEachAsync(2, async missingCurator => { var newBuzzUser = await _buzzAccountManager.AddBuzzUserAsync(new BuzzUser { MusicServiceId = (int)MusicService.AppleMusic, Username = missingCurator.CuratorId.ToString(), ImportPlaylists = false, CountryCode = AppleMusicPlaylistManager.DefaultStorefrontForTrackList, BuzzCategoryId = GetBuzzCategoryId(missingCurator) }); appleBuzzAccountsById.TryAdd(newBuzzUser.Username, newBuzzUser); }); _logger.Debug($"Loaded {appleBuzzAccountsById.Count} Buzz accounts ({appleBuzzAccountsById.Count(a => a.Value.BuzzCategoryId.HasValue && a.Value.BuzzCategoryId == (int)StaticBuzzCategory.AppleCurator)} apple curators " + $"and {appleBuzzAccountsById.Count(a => a.Value.BuzzCategoryId.HasValue && a.Value.BuzzCategoryId == (int)StaticBuzzCategory.Artists)} artists)."); return appleBuzzAccountsById.ToDictionary(k => k.Key, v => v.Value); } private int? GetBuzzCategoryId(CuratorIdWithType missingCurator) { if (string.IsNullOrWhiteSpace(missingCurator.CuratorType)) return null; if (missingCurator.CuratorType.Equals(AppleMusicCuratorType.AppleCurator, StringComparison.InvariantCultureIgnoreCase)) return (int?)StaticBuzzCategory.AppleCurator; if (missingCurator.CuratorType.Equals(AppleMusicCuratorType.Artists, StringComparison.InvariantCultureIgnoreCase)) return (int?)StaticBuzzCategory.Artists; return null; } private async Task UpdateApplePlaylistsAsync(List applePlaylists, List storefrontIds, Dictionary playlistMasterStorefront, DateTime currentUtcDate, List whitelistedPlaylists) { //var isMasterStorefront = true; _logger.Debug($"Getting tracklists for {storefrontIds.Count} storefronts."); using (var progress = new ConsoleProgressBar()) { var totalTracklistsToGet = (double)applePlaylists.Count * storefrontIds.Count; await applePlaylists.Batch(GetApplePlaylistsBatchSize()).ForEachAsync(1, async playlistBatch => { try { var playlists = playlistBatch.ToList(); foreach (var storeFront in storefrontIds) { var storefrontSpecific = playlists.Where(x => x.Storefront == storeFront).ToList(); if (!storefrontSpecific.Any()) { continue; } var appleApiPlaylistsResult = await ((Func, Task>>)_appleMusicApi.GetPlaylistsWithFullTracklistsAsync) .Tuple() .Timeout(TimeSpan.FromSeconds(5)) .Retry(2) .TryCatch() .OnFailure((tuple, result) => _logger.Error(result.Exception, $"Could not read playlists with tracklists from Apple. Batch size: {GetApplePlaylistsBatchSize()} Storefront: {storeFront} Playlists: {String.Join(",", storefrontSpecific.Select(p => p.Playlist.Id))}")) ((storeFront, storefrontSpecific.Select(p => p.Playlist.Id))); if (appleApiPlaylistsResult.IsFailed) { continue; } foreach (var appleApiPlaylist in appleApiPlaylistsResult.Value) { var playlist = playlists.Where(x => x.Playlist.Id == appleApiPlaylist.id && x.Storefront == storeFront).FirstOrDefault(); if (playlist != null) { var isMasterStorefront = (playlistMasterStorefront.GetValueOrDefault(playlist.Playlist.Id) == storeFront); await UpdatePlaylistWithTracklistAsync(storeFront, playlist.Playlist, appleApiPlaylist, isMasterStorefront, currentUtcDate, whitelistedPlaylists); } } SetProcessed(ref _appleCuratedPlaylistsProcess, totalTracklistsToGet, playlists.Count, progress); } } catch (Exception ex) { _logger.Error(ex, $"Error in update on apple-curated playlists."); } }); } } private static void SetProcessed(ref int processed, double total, int add, ConsoleProgressBar progress) { Interlocked.Add(ref processed, add); var percentage = processed / total; progress.Report(percentage); } private async Task UpdatePlaylistsInfoAsync(string storeFront, List playlists) { var appleApiLists = await _appleMusicApi.GetPlaylistsWithFullTracklistsAsync(storeFront, playlists.Select(p => p.Id)); await playlists.ForEachAsync(2, async playlist => { var apiList = appleApiLists.FirstOrDefault(p => p.id == playlist.Id); if (apiList == null) return; await UpdatePlaylistAsync(playlist, apiList); }); } private async Task UpdatePlaylistAsync(AppleMusicPlaylist playlist, ApplePlaylistData apiList, List whitelistedPlaylists = null) { playlist.Name = apiList.attributes.name; playlist.Artwork = apiList.attributes.artwork?.url; playlist.PlaylistType = apiList.attributes.playlistType; playlist.CuratorId = apiList.relationships?.curator?.data?.FirstOrDefault()?.id != null ? apiList.relationships?.curator?.data?.FirstOrDefault()?.id : whitelistedPlaylists?.FirstOrDefault(x => x.PlaylistId == playlist.Id)?.CuratorId; await _appleMusicPlaylistManager.UpdatePlaylistAsync(playlist); await AddOrUpdateCuratorAsync(apiList); } private async Task UpdatePlaylistWithTracklistAsync(string storeFront, AppleMusicPlaylist playlist, ApplePlaylistData apiPlaylist, bool isMasterStorefront, DateTime currentUtcDate, List whitelistedPlaylists) { var previousTracklist = await _appleMusicPlaylistManager.GetPlaylistCurrentTracklistAsync(apiPlaylist.id, storeFront); if (isMasterStorefront) { await UpdatePlaylistAsync(playlist, apiPlaylist, whitelistedPlaylists); } var tracklist = BuildTracklist(apiPlaylist, previousTracklist); await _appleMusicPlaylistManager.SetStorefrontSpecificDataAsync(storeFront, apiPlaylist.id, apiPlaylist.attributes.name, apiPlaylist.attributes.artwork?.url); if (IsTracklistChanged(tracklist, previousTracklist)) { await ((Func, string, Task>)_appleMusicPlaylistManager.SetPlaylistCurrentTracklistAndTrackStatisticsAsync) .Tuple() .ToUnit() .TryCatch() .OnFailure((tuple, result) => _logger.Error($"Could not update playlist current tracklist or update statistics for playlistId: '{playlist.Id}' storefront: {storeFront}")) .OnSuccess((tupel, result) => _logger.Info($"Updated playlist current tracklist and statistics for playlistId: '{playlist.Id}' storefront: {storeFront}")) ((playlist, tracklist, storeFront)); } if (!HasStoredHistoricPlaylists(storeFront, playlist.Id)) { await _appleMusicPlaylistManager.AddPlaylistHistoricTracklistAsync(playlist, tracklist, storeFront, currentUtcDate); } } private bool IsTracklistChanged(List tracklist, List previousTracklist) { if (tracklist.Count != previousTracklist.Count) { return true; } for (int playlistIndex = 0; playlistIndex < tracklist.Count; playlistIndex++) { var oldTrack = previousTracklist[playlistIndex]; var newTrack = tracklist[playlistIndex]; if (oldTrack.Id != newTrack.SongId || oldTrack.Added != newTrack.Added) { return true; } } return false; } private bool HasStoredHistoricPlaylists(string storeFront, string playlistId) { return _historicTracklistLookup.Contains(new Tuple(storeFront.ToLowerInvariant(), playlistId)); } private List BuildTracklist(ApplePlaylistData apiList, List previousTracklist) { var tracklist = apiList?.relationships?.tracks?.data?.ItemIndex().Select(trackApi => { var currentTrackApi = trackApi.Item; var currentPosition = trackApi.Index; var previousTrackWithMinPositionByIsrc = previousTracklist .Where(t => t.ISRC == currentTrackApi.attributes.isrc) .OrderBy(t => t.Position) .FirstOrDefault(); // If no previous track, set added to now, otherwise use the previous added time. var added = previousTrackWithMinPositionByIsrc != null ? previousTrackWithMinPositionByIsrc.Added : DateTime.UtcNow; // If position differs from latest recorded position, set the old position as "previous". Otherwise keep the old previous position. int? newPreviousPosition = null; DateTime? latestPositionChange = previousTrackWithMinPositionByIsrc?.LatestPositionChange; // Position has changed if (previousTrackWithMinPositionByIsrc != null && previousTrackWithMinPositionByIsrc.Position != currentPosition) { newPreviousPosition = previousTrackWithMinPositionByIsrc.Position; latestPositionChange = DateTime.UtcNow; } // Same position as before, leave everything as it was else if (previousTrackWithMinPositionByIsrc != null && previousTrackWithMinPositionByIsrc.Position == currentPosition) { newPreviousPosition = previousTrackWithMinPositionByIsrc.PreviousPosition; latestPositionChange = previousTrackWithMinPositionByIsrc.LatestPositionChange; } // Track is new in playlist, set utcnow as latest position change else if (previousTrackWithMinPositionByIsrc == null) { // New previous position is already set to null latestPositionChange = DateTime.UtcNow; } return new AppleMusicPlaylistTrack { Position = currentPosition, SongId = currentTrackApi.id, PreviousPosition = newPreviousPosition, Added = added, LatestPositionChange = latestPositionChange }; }).ToList(); return tracklist; } private async Task AddOrUpdateCuratorAsync(ApplePlaylistData apiPlaylist) { if (apiPlaylist?.relationships?.curator?.data?.FirstOrDefault() == null) return; var curatorData = apiPlaylist.relationships.curator.data.First(); if (!curatorData.id.HasValue) return; var curator = new AppleMusicCurator() { Id = curatorData.id.Value, Name = apiPlaylist.attributes.curatorName, CuratorType = curatorData.type, CuratorLink = curatorData.href }; await _appleMusicCuratorManager.AddOrUpdateCuratorAsync(curator); } private static string GetStorefront(AppleMusicPlaylist playlist, Dictionary appleMusicBuzzAccountsById) { return !playlist.CuratorId.HasValue ? AppleMusicPlaylistManager.DefaultStorefrontForTrackList : appleMusicBuzzAccountsById.ContainsKey(playlist.CuratorId.Value.ToString()) ? appleMusicBuzzAccountsById[playlist.CuratorId.Value.ToString()].CountryCode : AppleMusicPlaylistManager.DefaultStorefrontForTrackList; } private class CuratorIdWithType { public long CuratorId { get; set; } public string CuratorType { get; set; } } } }