using MoreLinq; using NLog; using Sony.Filtr.Buzz; using Sony.Filtr.Contracts.Definitions; using Sony.Filtr.Core.Buzz; using Sony.Filtr.ErrorLogging; using Sony.Filtr.Playlists; using Sony.Filtr.Playlists.Spotify; using Sony.Filtr.SpotifyWebAPI; using Sony.Filtr.SpotifyWebAPI.Model; using System; using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; using Sony.Filtr.Playlists.Spotify.Model; using System.Threading.Tasks.Dataflow; using System.Collections.ObjectModel; using Sony.Filtr.ApolloAPI.Models; using Sony.Filtr.ApolloAPI; namespace Sony.Filtr.Tasks.Tasks.Spotify { public class ImportBuzzPlaylists : IScheduledTask { private readonly BuzzManager _buzzManager; private readonly ScheduledTaskManager _scheduledTaskManager; private readonly IgnoredPlaylistsManager _ignoredPlaylistsManager; private readonly SpotifyPlaylistManager _spotifyPlaylistManager; private readonly BuzzAccountManager _buzzAccountManager; private readonly Logger _logger; private readonly IApolloWebApi _vendorApi; public ImportBuzzPlaylists(BuzzManager buzzManager, IApolloWebApi vendorApi, BuzzAccountManager buzzAccountManager, ScheduledTaskManager scheduledTaskManager, IgnoredPlaylistsManager ignoredPlaylistsManager, SpotifyPlaylistManager spotifyPlaylistManager) { _buzzManager = buzzManager; _vendorApi = vendorApi; _scheduledTaskManager = scheduledTaskManager; _ignoredPlaylistsManager = ignoredPlaylistsManager; _spotifyPlaylistManager = spotifyPlaylistManager; _buzzAccountManager = buzzAccountManager; _logger = NLog.LogManager.GetLogger("ImportBuzzPlaylists"); } public async Task ExecuteAsync(Guid scheduledTaskLogId) { return await ImportBuzzPlaylistsAsync(scheduledTaskLogId); } private async Task ImportBuzzPlaylistsAsync(Guid scheduledTaskLogId) { var taskLog = new ScheduledTaskLog(); var dataLog = new ScheduledTaskDataLog("ImportSpotifyBuzzPlaylist", DateTimeOffset.UtcNow, scheduledTaskLogId); var importUsers = await GetUsersToImportFromAsync(); var lastTaskInfo = await _scheduledTaskManager.GetLatestFinishedLogAsync("ImportBuzzPlaylists"); var lastImportDate = lastTaskInfo?.Finished ?? DateTimeOffset.Now.Date.AddDays(-1); var ignoredPlaylist = await _ignoredPlaylistsManager.GetIgnoredPlaylistsAsync(MusicService.Spotify); var ignoredPlaylistIds = Enumerable.ToHashSet(ignoredPlaylist.Select(p => p.PlaylistId).Distinct()); int totalProcessedPlaylistsCount = 0; var loadPlaylistsBlock = new TransformBlock(async userName => { IList playlists = null; try { playlists = await GetOwnPublicPlaylistsAsync(userName); System.Threading.Interlocked.Add(ref totalProcessedPlaylistsCount, playlists.Count); } catch (Exception ex) { _logger.Error(ex, $"Could not load Spotify playlists for {userName}"); ErrorLoggingManager.Instance.LogError(ex); } return new UserPlaylists(userName, playlists); }, new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 2 }); var savePlaylistsBlock = new ActionBlock(async userPlaylistsDto => { try { var playlists = userPlaylistsDto.Playlists.Where(p => !ignoredPlaylistIds.Contains(p.PlaylistId)).ToList(); if (playlists.Any()) { _logger.Debug($"Begin saving {playlists.Count()} playlists for user: {userPlaylistsDto.UserName}"); await _spotifyPlaylistManager.AddOrUpdateSpotifyPlaylistsForTrackingAsync(playlists); } else { _logger.Debug($"Could not find any playlists for user {userPlaylistsDto.UserName}"); } } catch (Exception ex) { _logger.Error(ex, $"Could not save Spotify playlists for {userPlaylistsDto.UserName}"); ErrorLoggingManager.Instance.LogError(ex); } }, new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 10 }); loadPlaylistsBlock.LinkTo(savePlaylistsBlock, new DataflowLinkOptions { PropagateCompletion = true }, dto => dto.Playlists != null); loadPlaylistsBlock.LinkTo(DataflowBlock.NullTarget(), new DataflowLinkOptions { PropagateCompletion = true }); _logger.Info($"Found {importUsers.Count} uses to process."); try { foreach (var userName in importUsers) { await loadPlaylistsBlock.SendAsync(userName); } loadPlaylistsBlock.Complete(); await savePlaylistsBlock.Completion; } catch (Exception ex) { _logger.Error(ex, "Exception: {0}", ex); ErrorLoggingManager.Instance.LogError(ex); } _logger.Info($"Ended ImportBuzzPlaylists. Total processed playlists: {totalProcessedPlaylistsCount}"); dataLog.Finished = DateTimeOffset.UtcNow; dataLog.Rows = importUsers.Count(); taskLog.DataLogs.Add(dataLog); return taskLog; } private async Task> GetUsersToImportFromAsync() { _logger.Info("Begin ImportBuzzPlaylists"); var oldBuzzUsers = _buzzManager.GetBuzzImportUsers(); _logger.Debug("Found " + oldBuzzUsers.Count() + " old import users."); List importFrom = new List(); importFrom.AddRange(oldBuzzUsers.Select(p => p.SpotifyUserName)); var buzzUsers = await _buzzAccountManager.GetBuzzUsersAsync(musicServiceId: (int)MusicService.Spotify); importFrom.AddRange(buzzUsers.Where(p => p.ImportPlaylists).Select(p => p.Username)); return importFrom.Distinct().ToList(); } private async Task> GetOwnPublicPlaylistsAsync(string spotifyUsername) { var result = new List(); _logger.Debug("Begin GetPublicPlaylists for user: {0}", spotifyUsername); const string fields = "items(id,name,owner.id,snapshot_id),total"; var userPlaylists = await _vendorApi.GetAllPublicPlaylistsAsync(spotifyUsername, fields); if (userPlaylists != null && userPlaylists.Any()) { _logger.Debug("Found {0} playlists for {1}", userPlaylists.Count(), spotifyUsername); var ownPlaylists = userPlaylists.Where(p => p.owner.id.Equals(spotifyUsername, StringComparison.InvariantCultureIgnoreCase)).ToList(); _logger.Debug("Found {0} own playlists for {1}", ownPlaylists.Count, spotifyUsername); var playlistReferences = ownPlaylists.Select(p => BuildPlaylistReference(p)).ToList(); result.AddRange(playlistReferences); } return result; } private SpotifyPlaylistTrackingReference BuildPlaylistReference(SpotifyPlaylistItem playlistItem) { return new SpotifyPlaylistTrackingReference() { PlaylistId = playlistItem.id, Name = playlistItem.name, User = playlistItem.owner?.id, SaveTracklist = true, SnapshotId = playlistItem.snapshot_id }; } private class UserPlaylists { public readonly string UserName; public readonly IReadOnlyCollection Playlists; public UserPlaylists(string userName, IList playlists) { this.UserName = userName; this.Playlists = playlists == null ? null : new ReadOnlyCollection(playlists); } } } }