using MoreLinq; using NLog; using Sony.Filtr.AppleMusic; using Sony.Filtr.AppleMusic.Data; using Sony.Filtr.AppleMusic.Helpers; using Sony.Filtr.AppleMusic.Playlists; using Sony.Filtr.Functional; using Sony.Filtr.Utility.Extensions; using System; using System.Collections.Concurrent; using System.Collections.Generic; using System.Linq; using System.Threading; using System.Threading.Tasks; namespace Sony.Filtr.Tasks.Tasks.AppleMusic { public class UpdateAppleMusicSongsTask : IScheduledTask { private readonly AppleMusicSongManager _appleMusicSongManager; private readonly AppleMusicPlaylistManager _appleMusicPlaylistManager; private readonly AppleMusicChartManager _appleMusicChartManager; private readonly AppleMusicUpdateHelper _appleMusicUpdateHelper; private readonly Logger _logger; private const int _MaxSongBatchSize = 300; public UpdateAppleMusicSongsTask(AppleMusicSongManager appleMusicSongManager, AppleMusicPlaylistManager appleMusicPlaylistManager, AppleMusicChartManager appleMusicChartManager, AppleMusicUpdateHelper appleMusicUpdateHelper) { _appleMusicSongManager = appleMusicSongManager; _appleMusicPlaylistManager = appleMusicPlaylistManager; _appleMusicChartManager = appleMusicChartManager; _appleMusicUpdateHelper = appleMusicUpdateHelper; _logger = LogManager.GetLogger("UpdateAppleMusicSongsTask"); } public async Task ExecuteAsync(Guid scheduledTaskLogId) { await AddMissingSongInformationAsync(); return null; } private async Task AddMissingSongInformationAsync() { var mergedStorefrontSongs = await IdentifySongsToFetchAsync(); // var mergedStorefrontSongs = new Dictionary>() { // { "us", new List(){ 31747, //35717, //35722, //35722, //35724, //35726, //35728, //35730, //35732, //35734, //35736, //35738, //35740, //35742, //38227, //38227, //38227, //38229, //38229} } // }; var allSongs = await GetSongsAsync(mergedStorefrontSongs); _logger.Debug(() => $"Marking songs as not available. Count: {allSongs.SongsToRemove}"); await _appleMusicSongManager.MarkSongsAsNotAvailable(allSongs.SongsToRemove); _logger.Debug(() => $"Saving songs and related data. Count: {allSongs.SongsToProcess}"); await _appleMusicUpdateHelper.SaveSongsWithRelatedDataAsync(allSongs.SongsToProcess, _logger); _logger.Debug(() => $"Recalculating songs with missing album or artist."); Func> recalculateSongs = async x => { await _appleMusicSongManager.RecalculateMissingAlbumOrArtistSongsTable(); return 1; }; await recalculateSongs .Timeout(TimeSpan.FromMinutes(120)) .Retry(1) .TryCatch() .OnFailure((x, result) => _logger.Error(() => $"Could not recalculate song missing data.")) (1); } private async Task>> IdentifySongsToFetchAsync() { Dictionary> chartSongs = new Dictionary>(); var chartDate = await _appleMusicChartManager.GetLatestChartDateAsync(); if (chartDate.HasValue) { chartSongs = await _appleMusicChartManager.GetSongIdsWithoutInformationByStorefrontAsync(chartDate.Value); _logger.Debug(() => $"For the latest chart date '{chartDate.Value.ToShortDateString()}' number of songs to fetch - {chartSongs.Sum(s => s.Value.Count())}"); } Func>>> getSongsMissingArtistOrAlbum = _appleMusicSongManager.GetSongsMissingDataAsync; var songsMissingArtistOrAlbum = await getSongsMissingArtistOrAlbum .Timeout(TimeSpan.FromMinutes(15)) .RetryWithCacellation(2) .TryCatch() .OnFailure(result => _logger.Error(() => $"Could not read songs with missing albums or artists from DB. Message: {result.Exception.GetFullMessage(ExceptionData.Message)}")) (); _logger.Debug(() => $"Songs missing artist or album - {songsMissingArtistOrAlbum.Value.Sum(v => v.Value.Count())}"); var playlistSongs = await _appleMusicPlaylistManager.GetSongIdsWithoutInformationByStorefrontAsync(); _logger.Debug(() => $"Songs without information by storefront - {playlistSongs.Sum(s => s.Value.Count())}"); var mergedStorefrontSongs = chartSongs .Union(playlistSongs) .Union(songsMissingArtistOrAlbum.IsOk ? songsMissingArtistOrAlbum.Value : new Dictionary>()) .GroupBy(p => p.Key, v => v.Value) .ToDictionary(k => k.Key, v => v.SelectMany(p => p).Distinct().ToList()); var count = mergedStorefrontSongs.Sum(s => s.Value.Count); _logger.Debug($"Got {count} missing songs to get information for."); return mergedStorefrontSongs; } public async Task<(AppleMusicSong[] SongsToProcess, (string storefront, long id)[] SongsToRemove)> GetSongsAsync(Dictionary> mergedStorefrontSongs) { ConcurrentBag songsToProcess = new ConcurrentBag(); ConcurrentBag<(string, long)> songsToRemove = new ConcurrentBag<(string, long)>(); foreach (var storefront in mergedStorefrontSongs) { _logger.Debug($"Fetching a total of {storefront.Value.Count} songs from API for {storefront.Key}."); try { var batches = storefront.Value.Batch(_MaxSongBatchSize).ToList(); var totalBatches = batches.Count; await batches.ForEachAsync(5, async songIdsBatch => { try { var songs = await _appleMusicUpdateHelper.FetchAppleMusicSongsAsync(storefront.Key, songIdsBatch); songs.ForEach(songsToProcess.Add); songIdsBatch.Except(songs.Select(s => s.Id)).ForEach(id => songsToRemove.Add((storefront.Key, id))); } catch (Exception ex) { _logger.Error(ex, "Could not load batch of songs"); } }); } catch (Exception ex) { _logger.Error(ex, $"Could not load songs from {storefront}"); } } _logger.Info(() => $"Initial songs count: {mergedStorefrontSongs.Sum(pair => pair.Value.Count)}. To remove: {songsToRemove.Count}"); return (songsToProcess.ToArray(), songsToRemove.ToArray()); } } }