using MoreLinq; using NLog; using Sony.Filtr.ApolloAPI; using Sony.Filtr.Contracts.Entities; using Sony.Filtr.NewMusicFriday; using Sony.Filtr.NewMusicFriday.Caching; using Sony.Filtr.Playlists.Models; using Sony.Filtr.Playlists.Spotify; using Sony.Filtr.SpotifyImages; using Sony.Filtr.SpotifyWebAPI.Model; using Sony.Filtr.Tasks.Helpers; using Sony.Filtr.Utility.Extensions; using System; using System.Collections.Concurrent; using System.Collections.Generic; using System.Linq; using System.Threading; using System.Threading.Tasks; using Sony.Filtr.VendorToSpotify; using Sony.Filtr.Functional; namespace Sony.Filtr.Tasks.Tasks.Spotify { public class ImportNewMusicFridayTask : IScheduledTask { private readonly SpotifyPlaylistManager _spotifyPlaylistManager; private readonly SpotifyAlbumManager _spotifyAlbumManager; private readonly SpotifyImageDownloader _spotifyImageHandler; private readonly UpdateAlbumsHelper _updateAlbumsHelper; private readonly NewMusicFridayManager _nmfManager; private readonly NewMusicFridayFactory _nmfFactory; private readonly NewMusicFridayCacheProvider _cacheProvider; private readonly UpdatePlaylistsHelper _updatePlaylistsHelper; private readonly Logger _logger; private readonly IApolloWebApi _vendorApi; private readonly ConcurrentDictionary _addedArtists = new ConcurrentDictionary(); private readonly ConcurrentDictionary _addedAlbums = new ConcurrentDictionary(); private readonly ConcurrentDictionary _addedTracks = new ConcurrentDictionary(); private readonly ConcurrentDictionary _addedAlbumCovers = new ConcurrentDictionary(); public ImportNewMusicFridayTask(SpotifyPlaylistManager spotifyPlaylistManager, SpotifyAlbumManager spotifyAlbumManager, SpotifyImageDownloader spotifyImageHandler, UpdatePlaylistsHelper updatePlaylistsHelper, UpdateAlbumsHelper updateAlbumsHelper, NewMusicFridayManager nmfManager, NewMusicFridayFactory nmfFactory, NewMusicFridayCacheProvider cacheProvider, IApolloWebApi vendorApi) { _spotifyPlaylistManager = spotifyPlaylistManager; _spotifyAlbumManager = spotifyAlbumManager; _spotifyImageHandler = spotifyImageHandler; _updatePlaylistsHelper = updatePlaylistsHelper; _updateAlbumsHelper = updateAlbumsHelper; _nmfManager = nmfManager; _nmfFactory = nmfFactory; _cacheProvider = cacheProvider; _logger = LogManager.GetLogger(nameof(ImportNewMusicFridayTask)); this._vendorApi = vendorApi; } public async Task ExecuteAsync(Guid scheduledTaskLogId) { //get playlists var nmfPlaylists = _nmfFactory.GetPlaylists().Where(p => p.PlaylistId == "37i9dQZF1DWT2SPAYawYcO").ToList(); var nmfPlaylistIndex = 0; var totalPlaylistCount = nmfPlaylists.Count; var precedingFriday = _nmfManager.GetPrecedingOrCurrentFriday(DateTime.UtcNow); await nmfPlaylists.ForEachAsync(10, async nmfPlaylist => { try { var affectedFriday = _nmfManager.GetLogicalFridayForImport(DateTime.UtcNow); _logger.Info($"Getting playlist {Interlocked.Increment(ref nmfPlaylistIndex)} of {totalPlaylistCount}"); //import data from spotify var fetchedPlaylist = (await this._vendorApi.GetPlaylistByIdWithAllTracksAsync(nmfPlaylist.PlaylistId)).ToSpotify(); if (fetchedPlaylist?.Tracks?.items != null && fetchedPlaylist.Tracks.items.Any()) { var currentTracksOnList = fetchedPlaylist.Tracks.items.Where(x => x?.track != null).ToList(); var fetchedNmfTracks = currentTracksOnList.ItemIndex().Select(x => new SpotifyNewMusicFridayPlaylistTrack() { PlaylistId = nmfPlaylist.PlaylistId, TrackId = x.Item.track.id, Date = affectedFriday, Added = x.Item.added_at, PlaylistIndex = x.Index }).ToList(); var tracksInDb = _nmfFactory.GetTracksForNewMusicFridayPlaylist(nmfPlaylist.PlaylistId, precedingFriday); if (ShouldPlaylistUpdate(tracksInDb, fetchedNmfTracks, affectedFriday)) { _logger.Debug($"Updating { nmfPlaylist.PlaylistId } with { fetchedNmfTracks.Count } tracks"); var tracksToUpdate = currentTracksOnList.ItemIndex().Select(x => _updatePlaylistsHelper.BuildSpotifyPlaylistTrack(x.Item, x.Index)).ToList(); await AddTracksWithRelatedDataAsync(tracksToUpdate); //ensure track data is available in DB _logger.Debug($"Setting { tracksToUpdate.Count } tracks for playlist { nmfPlaylist.PlaylistId }"); await _nmfFactory.SetPlaylistTracksForFriday(fetchedNmfTracks, nmfPlaylist.PlaylistId, affectedFriday); await AddAlbumCoversAsync(tracksToUpdate); await UpdatePlaylistImageAsync(fetchedPlaylist); await CacheNewDataAsync(nmfPlaylist.PlaylistId, affectedFriday); await SaveRegularTrackListAsync(nmfPlaylist.PlaylistId, tracksToUpdate, nmfPlaylist.Market); } } else { _logger.Warn($"No available data for NMF import. Playlist: { nmfPlaylist.PlaylistId }"); } } catch (Exception ex) { _logger.Error(ex, $"Error while importing data for playlist { nmfPlaylist?.PlaylistId }"); } }); return null; } private bool ShouldPlaylistUpdate(IList tracksInDb, IList fetchedNmfTracks, DateTime affectedFriday) { //Sometimes Spotify update their NMF lists Monday through Wednesday, and occationally leaves old tracks in, //so we attempt to filter them by checking if a percentage of tracks are added around this Friday const float filterThresholdPercentage = 0.8f; var newTracksThreshold = fetchedNmfTracks.Count * filterThresholdPercentage; if (tracksInDb.Count() == 0) { return true; } if (fetchedNmfTracks.Count(x => x.Added > affectedFriday.AddDays(-1)) > newTracksThreshold && _nmfManager.IsTrackListChanged(fetchedNmfTracks, tracksInDb)) { return true; } return false; } private async Task SaveRegularTrackListAsync(string playlistId, List tracksToUpdate, string countryCode) { var existingTracklist = await _spotifyPlaylistManager.GetTrackListAsync(playlistId); await _updatePlaylistsHelper.SetEarliestAddedDateAsync(tracksToUpdate, existingTracklist, playlistId, ownerUsername: "spotify"); await ((Func, string, Task>)_spotifyPlaylistManager.SetSpotifyTrackListAndUpdateStatistics) .Tuple() .ToUnit() .TryCatch() .OnFailure((tp, result) => _logger.Error(result.Exception, $"Could not update playlist current tracklist or statistics")) .OnSuccess((tp, result) => _logger.Info($"Updated playlist current tracklist and statistics for '{tp.Item1}'")) ((playlistId, tracksToUpdate, countryCode)); } private async Task CacheNewDataAsync(string playlistId, DateTime date) { var freshTracksData = await _nmfFactory.GetPlaylistTracksAsync(playlistId, date); _logger.Debug($"Setting new cache data for { playlistId } with { freshTracksData.Count } tracks"); _cacheProvider.SetPlaylistCollection(playlistId, date, freshTracksData); } private async Task AddAlbumCoversAsync(IEnumerable tracks) { var albumCoversToAdd = tracks.Where(t => !_addedAlbumCovers.ContainsKey(t.Track.Album.Id)).Select(t => t.Track.Album.Id).Distinct().ToList(); var threadSafeAlbumsToProcess = new List(); albumCoversToAdd.ForEach(x => { if (_addedAlbumCovers.TryAdd(x, 0)) threadSafeAlbumsToProcess.Add(x); }); if (threadSafeAlbumsToProcess.Any()) { var fetchedAlbums = await _spotifyAlbumManager.GetAlbumsAsync(threadSafeAlbumsToProcess); var albumsWithoutCovers = fetchedAlbums.Where(x => string.IsNullOrEmpty(x.SmallImageFilename)).Select(x => x.AlbumId).ToList(); if (albumsWithoutCovers.Any()) { _logger.Debug($"Updating album covers for { albumsWithoutCovers.Count } albums"); await _updateAlbumsHelper.SaveSpotifyAlbumImagesAsync(albumsWithoutCovers); } } } private async Task AddTracksWithRelatedDataAsync(ICollection playlistTracks) { _logger.Debug($"AddTracksWithRelatedDataAsync for {playlistTracks.Count} tracks"); var artists = playlistTracks.SelectMany(t => t.Track.Artists).Where(a => !string.IsNullOrWhiteSpace(a.Id)).DistinctBy(a => a.Id).ToList(); var artistToUpdate = artists.Where(a => !_addedArtists.ContainsKey(a.Id)).ToList(); await _spotifyPlaylistManager.AddSpotifyArtistsAsync(artistToUpdate); artistToUpdate.ForEach(a => _addedArtists.TryAdd(a.Id, 0)); var albums = playlistTracks.Select(t => t.Track.Album).Where(a => !string.IsNullOrWhiteSpace(a.Id)).DistinctBy(a => a.Id).ToList(); var albumsToUpdate = albums.Where(a => !_addedAlbums.ContainsKey(a.Id)).ToList(); await _spotifyPlaylistManager.AddSpotifyAlbumsAsync(albumsToUpdate); await UpdateAlbumInfo(albumsToUpdate.Select(x => x.Id).ToList()); albumsToUpdate.ForEach(t => _addedAlbums.TryAdd(t.Id, 0)); var tracksToUpdate = playlistTracks.Select(pt => pt.Track).Where(t => !_addedTracks.ContainsKey(t.Id)).ToList(); await _spotifyPlaylistManager.AddSpotifyTracksWithConnectionsAsync(tracksToUpdate); tracksToUpdate.ForEach(t => _addedTracks.TryAdd(t.Id, 0)); } private async Task UpdatePlaylistImageAsync(PlaylistResponse playlistResponse) { try { var imageUrl = playlistResponse.images?.FirstOrDefault()?.url; if (!string.IsNullOrEmpty(imageUrl)) { var imageFileName = await _spotifyImageHandler.StoreLargestPlaylistImageAsync(playlistResponse.id, imageUrl); var playlist = _spotifyPlaylistManager.GetPlaylistById(playlistResponse.id); if (imageFileName != null && playlist.Image != imageFileName) { _logger.Debug($"Updating playlist image for { playlist.PlaylistId } with value: { imageFileName }"); playlist.Image = imageFileName; _spotifyPlaylistManager.UpdatePlaylist(playlist); } } } catch (Exception e) { _logger.Error(e, $"Could not save image for playlist {playlistResponse?.id}"); } } private async Task UpdateAlbumInfo(ICollection albumIds) { _logger.Debug($"Saving album data for { albumIds.Count } albums"); var albums = (await this._vendorApi.GetAllAlbumInfoAsync(albumIds)).ToFullItem(); foreach (var album in albums) { await _spotifyAlbumManager.UpdateAlbumPropertiesAsync(album); } } } }