using MoreLinq; using NLog; using Sony.Filtr.ApolloAPI; using Sony.Filtr.Contracts.Abstractions; using Sony.Filtr.Contracts.Entities; using Sony.Filtr.Functional; using Sony.Filtr.PlaylistSynchronization.Data; using Sony.Filtr.SpotifyWebAPI; using Sony.Filtr.SpotifyWebAPI.Model; using Sony.Filtr.VendorToSpotify; using System; using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; namespace Sony.Filtr.PlaylistSynchronization.Synchronizer { public class SpotifySynchronizer : SynchronizerBase { private readonly SpotifyWebApi _spotifyWebApi_Default; private readonly IApolloWebApi _vendorApi; private readonly IServiceAccountManager _serviceAccountManager; private readonly Logger _logger; private readonly bool _safeMode = false; private readonly Func> GetAuthenticatedSpotifyApiForServiceAccountAsync; public SpotifySynchronizer(SpotifyWebApi spotifyWebApi, IServiceAccountManager serviceAccountManager, IApolloWebApi vendorApi) { _spotifyWebApi_Default = spotifyWebApi; _vendorApi = vendorApi; _serviceAccountManager = serviceAccountManager; _logger = LogManager.GetLogger("PlaylistSynchronization"); this.GetAuthenticatedSpotifyApiForServiceAccountAsync = Create_GetAuthenticatedSpotifyApiForServiceAccountFunc(); } private Func> Create_GetAuthenticatedSpotifyApiForServiceAccountFunc() { Func> f = async account => { var tuple = await this._serviceAccountManager.GetServiceAccountSpotifyClientIdAndSecretAsync(account.Id); return tuple.HasValue ? SpotifyWebApi.GetAuthenticatedSession(account, this._serviceAccountManager, tuple.Value.ClientId, tuple.Value.SecretId) : this._spotifyWebApi_Default.GetAuthenticatedSession(account, this._serviceAccountManager); }; return f.LazyMemoizeThreadSafe(); } protected override async Task CopyToPlaylistInternalAsync(GenericPlaylist genericPlaylist, Data.PlaylistSynchronization playlistSynchronization, ServiceAccount serviceAccount) { var playlistUri = new SpotifyLink(playlistSynchronization.ToPlaylistId); _logger.Debug("Begin copy from Spotify playlist {0} to spotify {1}", genericPlaylist.Name, playlistSynchronization.ToPlaylistId); var toSpotifyPlaylist = (await _vendorApi.GetPlaylistByIdWithAllTracksAsync(playlistUri.ExtractPlaylistID(), cacheExpiration: System.TimeSpan.FromSeconds(0))).ToSpotify(); if (toSpotifyPlaylist == null) { return PlaylistSynchronizationResult.FromError(SyncError.NoTargetPlaylist, $"Could not find playlist with id {playlistSynchronization.ToPlaylistId}"); } var authClient = await this.GetAuthenticatedSpotifyApiForServiceAccountAsync(serviceAccount); this._logger.Info($"For service account id: {serviceAccount.Id} ('{serviceAccount.UserIdentifier}') using client id '{authClient.ClientId?.Substring(0, 6)}'"); SyncCounts counts = await UpdatePlaylistTracksWithRange(toSpotifyPlaylist, toSpotifyPlaylist.Tracks.items, genericPlaylist, serviceAccount, authClient); if (playlistSynchronization.TitleCopyMode != FieldCopyMode.NoUpdate) { await UpdatePlaylistTitle(toSpotifyPlaylist, playlistSynchronization, genericPlaylist, authClient, serviceAccount); } return PlaylistSynchronizationResult.FromSuccess(counts, genericPlaylist.Tracks.Count); } private async Task UpdatePlaylistTitle(PlaylistResponse toSpotifyPlaylist, Data.PlaylistSynchronization playlistSynchronization, GenericPlaylist genericPlaylist, AuthenticatedSpotifyWebApi authClient, ServiceAccount serviceAccount) { var title = string.Empty; if (playlistSynchronization.TitleCopyMode == FieldCopyMode.CopySource) { _logger.Debug($"Updating title to {genericPlaylist.Name} from source"); title = genericPlaylist.Name; } else if (playlistSynchronization.TitleCopyMode == FieldCopyMode.UseSetting) { _logger.Debug($"Updating title to {playlistSynchronization.Title} according to setting"); title = playlistSynchronization.Title; } await authClient.SetPlaylistDetails(serviceAccount.UserIdentifier, toSpotifyPlaylist.id, title, toSpotifyPlaylist.@public); } private async Task UpdatePlaylistTracksWithRange(PlaylistResponse targetPlaylist, IEnumerable allPlaylistTrackItems, GenericPlaylist sourcePlaylist, ServiceAccount serviceAccount, AuthenticatedSpotifyWebApi spotifyApi) { var tracks = sourcePlaylist.Tracks.Count; var sourceTracks = sourcePlaylist.Tracks.DistinctBy(t => t.SpotifyUri).ToList(); _logger.Debug("In source playlist there are {0} tracks and {1} unique tracks", tracks, sourceTracks.Count); var trackDuplicatesToRemove = GetTrackDuplicatesToRemove(allPlaylistTrackItems, sourceTracks); _logger.Debug("In target playlist there are {0} duplicates which need to be removed individually: {1}", trackDuplicatesToRemove.Count, string.Join(",", trackDuplicatesToRemove.Select(t => t.uri))); var trackUrisToRemove = GetTracksToRemove(allPlaylistTrackItems, sourceTracks, trackDuplicatesToRemove); _logger.Debug("In target playlist there are {0} tracks which need to be removed: {1}", trackUrisToRemove.Count, string.Join(",", trackUrisToRemove.Select(t => t))); var trackUrisToAdd = GetTracksToAdd(allPlaylistTrackItems, sourceTracks); _logger.Debug("There are {0} tracks that need to be added: {1}", trackUrisToAdd.Count(), string.Join(",", trackUrisToAdd.Select(t => t))); string snapshot = targetPlaylist.snapshot_id; var localPlayListTracksIndex = GetLocalTracksToRemove(allPlaylistTrackItems); _logger.Debug("In target playlist there are {0} local tracks which need to be removed", localPlayListTracksIndex.Count()); if (trackDuplicatesToRemove.Any() || localPlayListTracksIndex.Any()) { _logger.Debug("Deleting local tracks."); _logger.Debug("Deleting track duplicates."); var trackPositions = trackDuplicatesToRemove.SelectMany(t => t.positions).Union(localPlayListTracksIndex).ToList(); if (!_safeMode) { snapshot = await spotifyApi.DeleteAllPlaylistTracksInPositionAsync(serviceAccount.UserIdentifier, targetPlaylist.id, trackPositions, snapshot); } } if (trackUrisToRemove.Any()) { _logger.Debug("Deleting tracks."); if (!_safeMode) { snapshot = await spotifyApi.DeleteMultiplePlaylistTracksAsync(serviceAccount.UserIdentifier, targetPlaylist.id, trackUrisToRemove, snapshot); } } if (trackUrisToAdd.Any()) { _logger.Debug("Adding tracks."); if (!_safeMode) { var addResponse = await spotifyApi.AddAllTrackAsync(serviceAccount.UserIdentifier, targetPlaylist.id, trackUrisToAdd); snapshot = addResponse.Last().snapshot_id; } } _logger.Debug("Getting all current tracks after modifying."); allPlaylistTrackItems = (await _vendorApi.GetAllPlaylistTracksByIdAsync(targetPlaylist.id, cacheExpiration: TimeSpan.FromSeconds(0))).Select(t => t.ToSpotify()).ToList(); allPlaylistTrackItems = allPlaylistTrackItems.Where(t => t.track != null).ToList(); var reorderedList = new List(); reorderedList.AddRange(allPlaylistTrackItems); var lastTrackInMovedRangeIndex = -1; foreach (var track in sourceTracks) { GenericTrack track1 = track; var trackInOrder = sourceTracks.Single(t => t.SpotifyUri == track1.SpotifyUri); var desiredPosition = sourceTracks.IndexOf(trackInOrder); if (desiredPosition <= lastTrackInMovedRangeIndex) continue; var trackInReorderedList = reorderedList.Single(t => t.track.uri == track1.SpotifyUri); var existingPosition = reorderedList.IndexOf(trackInReorderedList); var indexOfsourceTrack = desiredPosition; var indexOfreorderedListTrack = existingPosition; var range = 0; bool tryNext; do { range++; indexOfsourceTrack++; indexOfreorderedListTrack++; var nextTrackInSource = sourceTracks.Count > indexOfsourceTrack ? sourceTracks[indexOfsourceTrack] : null; var nextTrackInReordered = reorderedList.Count > indexOfreorderedListTrack ? reorderedList[indexOfreorderedListTrack] : null; if (nextTrackInSource != null && nextTrackInReordered != null && nextTrackInSource.SpotifyUri == nextTrackInReordered.track.uri) { lastTrackInMovedRangeIndex = indexOfsourceTrack; tryNext = true; } else { tryNext = false; } } while (tryNext); var rangeStart = existingPosition; var rangeLength = range; var insertBefore = desiredPosition; if (existingPosition != desiredPosition) { if (rangeLength == 1) { _logger.Debug("Moving track {0} from {1} to {2}", track.SpotifyUri, existingPosition, desiredPosition); } else { _logger.Debug("Moving track {0} and the following {1} tracks from {2} to {3}", track.SpotifyUri, rangeLength, existingPosition, desiredPosition); } int rangeStartModified = rangeStart; var insertBeforeModified = insertBefore; foreach (var moveBatch in Enumerable.Range(0, rangeLength).Batch(100)) { var moveCount = moveBatch.Count(); if (!_safeMode) { var orderResponse = await spotifyApi.OrderPlaylistTracksAsync(serviceAccount.UserIdentifier, targetPlaylist.id, rangeStartModified, moveCount, insertBeforeModified, snapshot); snapshot = orderResponse.snapshot_id; } rangeStartModified = rangeStartModified + moveCount; insertBeforeModified = insertBeforeModified + moveCount; } reorderedList = Move(reorderedList, existingPosition, desiredPosition, range); } } _logger.Info("Done with playlist sync for {0}", targetPlaylist.uri); return new SyncCounts(trackUrisToAdd.Count(), trackDuplicatesToRemove.Count, trackUrisToRemove.Count); } private List GetTrackDuplicatesToRemove(IEnumerable allPlaylistTrackItems, IEnumerable sourcePlaylistTracks) { List trackInPositions = new List(); var allPlTrackItems = allPlaylistTrackItems != null ? allPlaylistTrackItems.ToList() : new List(); var trackLookup = sourcePlaylistTracks.Select(t => t.SpotifyUri).ToHashSet(); var trackGroups = allPlTrackItems.Where(t => trackLookup.Contains(t.track.uri)).GroupBy(g => g.track.uri); foreach (var duplicateTrackGroup in trackGroups.Where(c => c.Count() > 1)) { var orderedDuplicateTracks = duplicateTrackGroup.OrderBy(r => r.added_at); //Identify the position of the tracks, leave the oldest var positions = orderedDuplicateTracks.Skip(1).Select(t => allPlTrackItems.IndexOf(t)).ToArray(); trackInPositions.Add(new TrackInPosition() { uri = duplicateTrackGroup.Key, positions = positions }); } return trackInPositions; } private IEnumerable GetLocalTracksToRemove(IEnumerable allTracksItem) { var allPlTrackItems = allTracksItem != null ? allTracksItem.ToList() : new List(); var localPlayListTracks = allPlTrackItems.Where(item => item.is_local); return localPlayListTracks.Select(t => allPlTrackItems.IndexOf(t)).ToArray(); } private static IEnumerable GetTracksToAdd(IEnumerable currentTracks, IEnumerable sourcePlaylistTracks) { return sourcePlaylistTracks.Select(t => t.SpotifyUri).Except(currentTracks.Select(ti => ti.track.uri)).ToArray(); } private static List GetTracksToRemove(IEnumerable currentTracks, IEnumerable sourcePlaylistTracks, IEnumerable trackDuplicatesToRemove) { return currentTracks .Where(item => !item.is_local) .Select(t => t.track.uri) .Except(sourcePlaylistTracks.Select(ti => ti.SpotifyUri)) .Except(trackDuplicatesToRemove.Select(t => t.uri)).ToList(); } private List Move(List list, int oldIndex, int newIndex, int range) { var items = list.GetRange(oldIndex, range); list.RemoveRange(oldIndex, range); list.InsertRange(newIndex, items); return list; } } }