using MoreLinq; using NLog; using Sony.Filtr.Buzz; using Sony.Filtr.Contracts.Definitions; using Sony.Filtr.Contracts.Entities.Buzz; using Sony.Filtr.Core.SpotifyUserCountries; using Sony.Filtr.Playlists.Models; using Sony.Filtr.Playlists.Spotify; using Sony.Filtr.Playlists.Spotify.Model.History; using Sony.Filtr.Tasks.Helpers; using Sony.Filtr.Utility; using Sony.Filtr.Utility.Extensions; using Sony.Filtr.Functional; using System; using System.Collections.Concurrent; using System.Collections.Generic; using System.IO; using System.Linq; using System.Text; using System.Threading; using System.Threading.Tasks; namespace Sony.Filtr.Tasks.Tasks { public abstract class UpdatePlaylistInfoBase : IScheduledTask { public const string PlaylistWithInvalidTracksCorruptedSnapshotId = "HasInvalidTracks"; protected readonly SpotifyPlaylistManager _spotifyPlaylistManager; protected readonly SpotifyPlaylistHistoricTrackListManager _historicTrackListManager; protected readonly UpdatePlaylistsHelper _updatePlaylistsHelper; protected readonly BuzzAccountManager _buzzAccountManager; protected readonly SpotifyUserCountryManager _spotifyUserCountryManager; protected readonly TimeSpan ApolloCache; private readonly PlaylistUpdateTempData _updateRunData = new PlaylistUpdateTempData(); private readonly HashSet _saveHistoryForBuzzCategories = new HashSet() { (int)StaticBuzzCategory.SonyMusic, (int)StaticBuzzCategory.Spotify, (int)StaticBuzzCategory.Tastemakers, }; private Dictionary UserCountriesMap; private Dictionary BuzzUsersMapByName; private ConcurrentDictionary PlaylistIdToSaveModesMap; private readonly ActionTracker errorStatisticsTracker; public UpdatePlaylistInfoBase( SpotifyPlaylistManager spotifyPlaylistManager, SpotifyPlaylistHistoricTrackListManager historicTrackListManager, UpdatePlaylistsHelper updatePlaylistsHelper, BuzzAccountManager buzzAccountManager, SpotifyUserCountryManager spotifyUserCountryManager) { this._spotifyPlaylistManager = spotifyPlaylistManager; this._historicTrackListManager = historicTrackListManager; this._updatePlaylistsHelper = updatePlaylistsHelper; this._buzzAccountManager = buzzAccountManager; this._spotifyUserCountryManager = spotifyUserCountryManager; this.errorStatisticsTracker = new ActionTracker(Path.Combine("logs", $"errorStatistics_{DateTime.Now.ToString("yyyy_MM_dd")}.txt"), 1); this.ApolloCache = UseApolloCacheFromConfig() ? TimeSpan.FromHours(23) : TimeSpan.FromSeconds(0); } protected virtual bool PreloadAlbumsAndArtists => Maybe.GetAppSettingsBooleanOrDefault("UpdatePlaylistInfo_PreloadAlbumsAndArtists", true); public async Task ExecuteAsync(Guid scheduledTaskLogId) { if (this.PreloadAlbumsAndArtists) { await Task.WhenAll( this._spotifyPlaylistManager.GetAllAlbumIdsAsync() .ContinueWith(idsTask => idsTask.Result.ForEach(id => this._updateRunData.KnownAlbums.TryAdd(id, 0))), this._spotifyPlaylistManager.GetAllArtistIdsAsync() .ContinueWith(idsTask => idsTask.Result.ForEach(id => this._updateRunData.KnownArtists.TryAdd(id, 0))) ); } this.UserCountriesMap = this._spotifyUserCountryManager.GetSpotifyUsersCountry().ToDictionary(k => k.SpotifyUserName, v => v.CountryCode); this.BuzzUsersMapByName = (await _buzzAccountManager.GetBuzzUsersAsync(musicServiceId: (int)MusicService.Spotify)).ToDictionary(k => k.Username, v => v); this.PlaylistIdToSaveModesMap = new ConcurrentDictionary(_historicTrackListManager.GetCustomTracklistHistoryModes().ToDictionary(k => k.PlaylistId, v => v.SaveMode)); CancellationTokenSource cts = new CancellationTokenSource(); var errorStatisticsThread = StartErrorStatisticsThread(cts.Token); ScheduledTaskLog taskLog = null; try { taskLog = await ExecuteJobAsync(scheduledTaskLogId); } catch (Exception ex) { this.LogError(ex); throw; } finally { cts.Cancel(); await errorStatisticsThread; await this.errorStatisticsTracker.CompleteAsync(); } return taskLog; } protected BuzzUser GetBuzzUserByNameOrDefault(string userName) { return this.BuzzUsersMapByName.GetValueOrDefault(userName); } protected void GenerateProgressChart(ProgressTracker tracker, string filePath) { try { using (var chart = new ProgressChart(tracker)) { //chart.ToChartBitmap($"logs/ProgressChart_{DateTime.Now.ToString("yyyy_MM_dd")}.bmp"); chart.ToChartBitmap(filePath); } } catch (Exception ex) { this.LogError(ex, () => $"Could not generate progress chart '{filePath}'"); } } protected abstract Task ExecuteJobAsync(Guid scheduledTaskLogId); private ConcurrentBag<(Exception, string, DateTime)> errors = new ConcurrentBag<(Exception, string, DateTime)>(); private void SaveErrorStatistics(Exception ex, Func getMessage) { if (getMessage != null) { errors.Add((ex, getMessage(), DateTime.Now)); } } protected void LogError(Exception ex, Func getMessage = null) { this.LogErrorOnly(ex, getMessage); this.SaveErrorStatistics(ex, getMessage); } protected void LogError(Func getMessage) { this.LogError(new Exception(), getMessage); } protected abstract void LogErrorOnly(Exception ex, Func getMessage); private Task StartErrorStatisticsThread(CancellationToken cancellationToken) { return TaskHelper.CreateLongRunningLoggingThread( cancellationToken, () => { if (!this.errors.Any()) { return String.Empty; } StringBuilder builder = new StringBuilder(); builder.AppendLine("******************Exception statistics******************"); builder.AppendLine($"Total exceptions: {this.errors.Count()}"); foreach (var exceptionGroup in this.errors.GroupBy(tuple => (Type: tuple.Item1.GetType(), Message: tuple.Item1.Message))) { builder.AppendLine($"{exceptionGroup.Key.Type} '{exceptionGroup.Key.Message}': ({exceptionGroup.Count()})"); foreach (var messageGroup in exceptionGroup.GroupBy(eg => eg.Item2)) { builder.AppendLine($"\t{messageGroup.Key}"); builder.AppendLineForEach(messageGroup.Select(g => g.Item3).OrderBy(dt => dt), dt => $"\t\t{dt}"); } } return builder.ToString(); }, this.errorStatisticsTracker, TimeSpan.FromMinutes(1)); } protected static void MarkPlaylistAsHavingInvalidTracks(ApolloAPI.Models.SpotifyPlaylist playlist) { playlist.snapshot_id = PlaylistWithInvalidTracksCorruptedSnapshotId; playlist.HasSnapshotIdChanged = true; } private static bool UseApolloCacheFromConfig() { return Maybe.GetAppSettingsBooleanOrDefault("UseApolloCache", false); } private static bool WriteResponsesWithCorruptedTracks() { return Maybe.GetAppSettingsBooleanOrDefault("WriteResponsesWithCorruptedTracks", false); } protected async Task VerifyAndHandlePlaylistInvalidTrack(ApolloAPI.Models.SpotifyPlaylist spotifyPlaylist, Action invalidTracksSummaryStringAction) { Func)>>> getInvalidTracks = async pl => { var emptyResult = Result<(ApolloAPI.Models.SpotifyPlaylist, IEnumerable<(ApolloAPI.Models.SpotifyPlaylistTrackItem Item, int Index)>)>.FromException(new Exception("No invalid tracks")); if (pl == null) { return emptyResult; } var invalidTracks = pl.Tracks.items .Select((item, index) => (Item: item, Index: index)) .Where(tuple => !ApolloAPI.Models.SpotifyPlaylist.IsValidTrack(tuple.Item)) .ToList(); if (!invalidTracks.Any()) { return emptyResult; } if (!this.ShouldSaveTracklistHistory(pl)) { return emptyResult; } return (pl, invalidTracks); }; Func<(ApolloAPI.Models.SpotifyPlaylist, IEnumerable<(ApolloAPI.Models.SpotifyPlaylistTrackItem Item, int Index)>), Result> getInvalidTracksSummary = tuple => { IEnumerable<(ApolloAPI.Models.SpotifyPlaylistTrackItem Item, int Index)> invalidTracks = tuple.Item2; StringBuilder invalidTrackMessageBuilder = new StringBuilder(); invalidTrackMessageBuilder.AppendLine($"Found {invalidTracks.Count()} invalid tracks in playlist '{spotifyPlaylist.id}'"); invalidTrackMessageBuilder.AppendLine($"Invalid tracks indexes: {String.Join(", ", invalidTracks.Select(i => i.Index))}"); foreach (var invalidTrack in invalidTracks) { invalidTrackMessageBuilder.AppendLine($"Track at index {invalidTrack.Index} name: '{invalidTrack.Item.track?.name}': "); if (invalidTrack.Item.track == null) { invalidTrackMessageBuilder.Append("track is NULL "); } else { if (String.IsNullOrWhiteSpace(invalidTrack.Item.track.id)) { invalidTrackMessageBuilder.Append($"invalid id - '{invalidTrack.Item.track.id}' "); } if (String.IsNullOrWhiteSpace(invalidTrack.Item.track.uri)) { invalidTrackMessageBuilder.Append($"invalid uri - '{invalidTrack.Item.track.uri}' "); } } invalidTrackMessageBuilder.AppendLine(); } if (WriteResponsesWithCorruptedTracks() && spotifyPlaylist.RawResponses != null) { string playlistRawRequestsFileName = $"{spotifyPlaylist.id}_{DateTime.Now.ToString("yyyy_MM_dd_HH_mm")}.json"; var directory = Path.Combine("invalid_playlist_tracks", $"{DateTime.Now.ToString("yyyy_MM_dd")}"); if (!Directory.Exists(directory)) { Directory.CreateDirectory(directory); } File.WriteAllLines( Path.Combine(directory, playlistRawRequestsFileName), spotifyPlaylist.RawResponses.Select(pair => $"{pair.Key}{Environment.NewLine}{pair.Value.JsonPrettify()}").ToArray()); invalidTrackMessageBuilder.AppendLine($"Raw playlist response(s) saved to file '{playlistRawRequestsFileName}'"); } return invalidTrackMessageBuilder.ToString().ToResult(); }; await getInvalidTracks(spotifyPlaylist) .TapAsync(tuple => MarkPlaylistAsHavingInvalidTracks(tuple.Item1)) .Bind(getInvalidTracksSummary) .TapAsync(invalidTracksSummaryStringAction); } protected async Task AddNewSpotifyArtistsIfAnyAsync(IEnumerable playlistTracks) { var artists = playlistTracks .Where(t => t.Track.Artists != null) .SelectMany(t => t.Track.Artists) .Where(a => !string.IsNullOrWhiteSpace(a.Id)) .DistinctBy(a => a.Id) .ToList(); var artistToUpdate = artists.Where(a => !_updateRunData.KnownArtists.ContainsKey(a.Id)).ToList(); await _spotifyPlaylistManager.AddSpotifyArtistsAsync(artistToUpdate); artistToUpdate.ForEach(a => _updateRunData.KnownArtists.TryAdd(a.Id, 0)); } protected async Task AddNewSpotifyAlbumsIfAnyAsync(IEnumerable playlistTracks) { var albums = playlistTracks .Where(t => t.Track.Album != null) .Select(t => t.Track.Album) .Where(a => !string.IsNullOrWhiteSpace(a.Id)) .DistinctBy(a => a.Id) .ToList(); var albumsToUpdate = albums.Where(a => !_updateRunData.KnownAlbums.ContainsKey(a.Id)).ToList(); await _spotifyPlaylistManager.AddSpotifyAlbumsAsync(albumsToUpdate); albumsToUpdate.ForEach(t => _updateRunData.KnownAlbums.TryAdd(t.Id, 0)); } protected async Task AddTracksWithRelatedDataAsync(IEnumerable playlistTracks) { await this.AddNewSpotifyArtistsIfAnyAsync(playlistTracks); await this.AddNewSpotifyAlbumsIfAnyAsync(playlistTracks); var tracksToUpdate = playlistTracks .Select(pt => pt.Track) .Where(t => !_updateRunData.AddedTracks.ContainsKey(t.Id)) .ToList(); await _spotifyPlaylistManager.AddSpotifyTracksWithConnectionsAsync(tracksToUpdate); tracksToUpdate.ForEach(t => _updateRunData.AddedTracks.TryAdd(t.Id, 0)); } protected bool ShouldSaveTracklistHistory(ApolloAPI.Models.SpotifyPlaylist playlist) { if (playlist == null) { return false; } var user = this.BuzzUsersMapByName.GetValueOrDefault(playlist.owner?.id); return this.ShouldSaveTracklistHistory(user?.BuzzCategoryId, playlist.id); } protected bool ShouldSaveTracklistHistory(int? buzzCategoryId, string playlistId) { return this.ShouldSaveTracklistHistory(buzzCategoryId, this.PlaylistIdToSaveModesMap.GetValueOrDefault(playlistId)); } private bool ShouldSaveTracklistHistory(int? buzzCategoryId, TracklistHistorySaveMode playlistMode) { switch (playlistMode) { case TracklistHistorySaveMode.Enabled: return true; case TracklistHistorySaveMode.Disabled: return false; case TracklistHistorySaveMode.Automatic: return ShouldSaveTracklistHistoryForCategory(buzzCategoryId); default: //Unknown format, let's save to be sure we don't miss anything. return true; } } protected string GetCountry(string dbPlaylistCountryCode, BuzzUser buzzUser) { if (!String.IsNullOrWhiteSpace(dbPlaylistCountryCode)) { return dbPlaylistCountryCode; } if (!String.IsNullOrWhiteSpace(buzzUser?.CountryCode)) { return buzzUser.CountryCode; } if (buzzUser != null) { return this.UserCountriesMap.GetValueOrDefault(buzzUser.Username); } return null; } private bool ShouldSaveTracklistHistoryForCategory(int? buzzCategory) { return buzzCategory.HasValue && _saveHistoryForBuzzCategories.Contains(buzzCategory.Value); } } }