using MoreLinq; using NLog; using Sony.Filtr.ApolloAPI; using Sony.Filtr.Contracts.Abstractions; using Sony.Filtr.Contracts.Definitions; using Sony.Filtr.Playlists.Spotify; using Sony.Filtr.Playlists.Spotify.Model; using Sony.Filtr.Utility; using Sony.Filtr.Utility.Extensions; using Sony.Filtr.Functional; using System; using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; using System.Threading.Tasks.Dataflow; namespace Sony.Filtr.Tasks.Tasks.Spotify { public class NewPlaylistsImporter { private readonly NewPlaylistsReader Reader; private readonly SpotifyPlaylistManager _spotifyPlaylistManager; private readonly IBuzzAccountManager _buzzAccountManager; private readonly IApolloWebApi _vendorApi; private readonly Logger _logger; private PlaylistData _filtrPlaylistData; public NewPlaylistsImporter( NewPlaylistsReader reader, SpotifyPlaylistManager spotifyPlaylistManager, IBuzzAccountManager buzzAccountManager, IApolloWebApi vendorApi) { this.Reader = reader; this._spotifyPlaylistManager = spotifyPlaylistManager; this._buzzAccountManager = buzzAccountManager; this._vendorApi = vendorApi; _logger = LogManager.GetLogger("NewPlaylistsImporter"); } public async Task ImportAsync() { this._filtrPlaylistData = await this.GetPlaylistDataAsync(); _logger.Info($"Found {this._filtrPlaylistData.AllPlaylists.Count} known playlist ids in DB"); Func isNewPlaylist = id => !this._filtrPlaylistData.AllPlaylists.Contains(id); TransformBlock filterOnlyNewPlaylistByIdBlock = isNewPlaylist .ToTask() .Map( inputTransform: id => id, outputTransform: (id, isNew, id2) => isNew ? id : null ) .AsTransformBlock(new ExecutionDataflowBlockOptions() { MaxDegreeOfParallelism = 1, BoundedCapacity = 100 }); Func> getPlaylistFromSpotify = GetPlaylistFromSpotify; TransformBlock> getPlaylistBlock = getPlaylistFromSpotify .Timeout(TimeSpan.FromSeconds(10)) .Retry(1) .LazyMemoizeThreadSafeFirstOrDefault(Task.FromResult((SpotifyPlaylistTrackingReference)null)) .TryCatch() .OnSuccess((id, result) => { if (result.Value != null) { _logger.Info($"New playlist detected '{id}'"); } }) .OnFailure((id, result) => _logger.Error(result.Exception, $"Could not load playlist from Spotify '{id}'")) .AsTransformBlock(new ExecutionDataflowBlockOptions() { MaxDegreeOfParallelism = GetFromSpotifyCount() }); BatchBlock> addOrUpdateBatchBlock = new BatchBlock>(100); Func, Task> addOrUpdatePlaylists = _spotifyPlaylistManager.AddOrUpdateSpotifyPlaylistsForTrackingAsync; ActionBlock>> addOrUpdatePlaylistsBlock = addOrUpdatePlaylists .ToUnit() .Timeout(TimeSpan.FromSeconds(15)) .Retry(1) .Map>, Unit, IEnumerable, Unit>( inputTransform: dto => dto.Where(p => p.Value != null).Select(d => d.Value).ToList(), outputTransform: (refs, unit, dto) => Unit.Default ) .TryCatch() .OnFailure((dto, result) => _logger.Error(result.Exception, $"Could not add or update playlists")) .OnSuccess((playlistResult, result) => _logger.Info($"Playlists updated {String.Join(",", playlistResult.Where(p => p.Value != null).Select(r => r.Value.PlaylistId))}")) .AsActionBlock(new ExecutionDataflowBlockOptions() { MaxDegreeOfParallelism = 1 }); var linkOptions = new DataflowLinkOptions() { PropagateCompletion = true }; filterOnlyNewPlaylistByIdBlock.LinkTo(getPlaylistBlock, linkOptions, dto => dto != null); filterOnlyNewPlaylistByIdBlock.LinkTo(DataflowBlock.NullTarget()); getPlaylistBlock.LinkTo(addOrUpdateBatchBlock, linkOptions, dto => dto.IsOk && (dto.Value != null)); getPlaylistBlock.LinkTo(DataflowBlock.NullTarget>()); addOrUpdateBatchBlock.LinkTo(addOrUpdatePlaylistsBlock, linkOptions); _logger.Info("Starting to process"); foreach (var id in this.Reader.GetPlaylistIds()) { await filterOnlyNewPlaylistByIdBlock.SendAsync(id); } filterOnlyNewPlaylistByIdBlock.Complete(); await addOrUpdatePlaylistsBlock.Completion; var afterImportData = await this.GetPlaylistDataAsync(); _logger.Info($"{afterImportData.AllPlaylists.Count - this._filtrPlaylistData.AllPlaylists.Count} new playlists imported"); } private async Task GetPlaylistFromSpotify(string playlistId) { var playlist = await _vendorApi.GetSpotifyPlaylistByIdAsync(playlistId, cacheExpiration: TimeSpan.FromSeconds(0), fields: "name,owner.id,snapshot_id"); SpotifyPlaylistTrackingReference returnValue = null; if (playlist != null) { _logger.Info($"'{playlistId}' {playlist.GetSummary()}"); var saveTracklist = _filtrPlaylistData.BuzzImportUsers.Contains(playlist.owner.id); returnValue = new SpotifyPlaylistTrackingReference() { PlaylistId = playlistId, Name = playlist.name, User = playlist.owner.id, SaveTracklist = saveTracklist, SnapshotId = playlist.snapshot_id }; } else { _logger.Info($"****** Received NULL from Spotify - '{playlistId}'"); } return returnValue; } private async Task GetPlaylistDataAsync() { var allPlaylists = (await _spotifyPlaylistManager.GetAllPlaylistIdsAsync(includeRemoved: true)).ToHashSet(); var allBuzzUsers = await _buzzAccountManager.GetBuzzUsersAsync(musicServiceId: (int)MusicService.Spotify); var buzzImportUsers = allBuzzUsers.Where(p => p.BuzzCategoryId == (int)StaticBuzzCategory.Spotify || p.BuzzCategoryId == (int)StaticBuzzCategory.SonyMusic).Select(p => p.Username).ToHashSet(StringComparer.InvariantCultureIgnoreCase); return new PlaylistData { AllPlaylists = allPlaylists, BuzzImportUsers = buzzImportUsers }; } private static int GetFromSpotifyCount() { return Maybe.GetAppSettingsIntOrDefault("NewPlaylistsImporter_Get_From_Spotify_Block_Count", 2); } private class PlaylistData { public HashSet AllPlaylists { get; set; } public HashSet BuzzImportUsers { get; set; } } } }