using System; using System.Collections.Generic; using System.Linq; using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; using AngleSharp.Extensions; using AngleSharp.Parser.Html; using MoreLinq; using NLog; using Sony.Filtr.Contracts.Definitions; using Sony.Filtr.Contracts.Entities; using Sony.Filtr.PlaylistSynchronization.Data; using Sony.Filtr.Utility; using Sony.Filtr.Utility.Extensions; using Sony.Filtr.YouTube.Api; namespace Sony.Filtr.PlaylistSynchronization.Synchronizer { public class YoutubeSynchronizer : SynchronizerBase { private readonly YouTubeApi _youtubeApi; private readonly SynchronizationTrackManager _synchronizationTrackManager; private readonly FlaggedChannelsManager _flaggedChannelsManager; private readonly Logger _logger; private readonly List _officialLookingSuffixes = new List() { "official", "officiel", "offiziell", "ufficiale", "offisiell", "oficial", "officiell", "channel", }; private readonly List _blacklistedPartialPhrases = new List() { "remaster", "album", "full length", }; private readonly List _blacklistedExactPhrases = new List() { "single version", "radio edit", "single edit", "radio version", "radio mix", "single mix", "stereo version", "extended version", "original mix", "single remix", }; private readonly List _blacklistedYouTubeTitleWords = new List() { "karaoke", "behind the scenes" }; public YoutubeSynchronizer(YouTubeApi youtubeApi, SynchronizationTrackManager synchronizationTrackManager, FlaggedChannelsManager flaggedChannelsManager) { _youtubeApi = youtubeApi; _synchronizationTrackManager = synchronizationTrackManager; _flaggedChannelsManager = flaggedChannelsManager; _logger = LogManager.GetLogger("PlaylistSynchronization"); } protected override async Task CopyToPlaylistInternalAsync(GenericPlaylist genericPlaylist, Data.PlaylistSynchronization playlistSynchronization, ServiceAccount toAccount) { _logger.Debug("Begin copy from Spotify playlist {0} to youtube: {1}", genericPlaylist.Name, playlistSynchronization.ToPlaylistId); var authClient = _youtubeApi.GetAuthenticatedYoutubeApi(toAccount); SyncCounts counts = await UpdatePlaylistTracks(playlistSynchronization, genericPlaylist, authClient); return PlaylistSynchronizationResult.FromSuccess(counts, genericPlaylist.Tracks.Count); } public async Task UpdatePlaylistTracks(Data.PlaylistSynchronization playlistSynchronization, GenericPlaylist sourcePlaylist, AuthenticatedYoutubeApi authClient) { var targetPlaylistItems = await authClient.GetAllPlaylistItemsAsync(playlistSynchronization.ToPlaylistId); if (targetPlaylistItems == null) { throw new PlaylistSynchronizationException($"Could not get youtube playlist items for {playlistSynchronization.ToPlaylistId}", SyncError.NoTargetPlaylist); } var sourceTracks = sourcePlaylist.Tracks.DistinctBy(t => t.SpotifyUri).ToList(); var sourceSynchronizationTracks = await GetTrackMappings(sourceTracks, authClient); sourceSynchronizationTracks = await InsertSelectedVideos(sourceSynchronizationTracks, playlistSynchronization, authClient); var duplicateItemsToRemove = GetTrackDuplicatesToRemove(targetPlaylistItems, sourceSynchronizationTracks); _logger.Debug("In target playlist there are {0} duplicate videos which need to be removed individually: {1}", duplicateItemsToRemove.Count, string.Join(",", duplicateItemsToRemove.Select(t => t.snippet.resourceId.videoId))); var itemsToRemove = GetPlaylistItemsToRemove(targetPlaylistItems, sourceSynchronizationTracks, duplicateItemsToRemove); _logger.Debug("In target playlist there are {0} videos which need to be removed: {1}", itemsToRemove.Count, string.Join(",", itemsToRemove.Select(t => t.snippet.title))); var videosToAdd = GetVideosToAdd(targetPlaylistItems, sourceSynchronizationTracks); _logger.Debug("There are {0} videos that need to be added: {1}", videosToAdd.Count, string.Join(",", videosToAdd.Select(t => t))); if (duplicateItemsToRemove.Any()) { await authClient.DeletePlaylistItemsAsync(duplicateItemsToRemove); } if (videosToAdd.Any()) { foreach (var videoId in videosToAdd) { var youtubeId = new Id() { videoId = videoId }; var desiredPosition = sourceSynchronizationTracks.ItemIndex().Single(p => p.Item.TrackId == videoId).Index; await authClient.AddVideoToPlaylistAsync(playlistSynchronization.ToPlaylistId, youtubeId, desiredPosition); } } if (itemsToRemove.Any()) { await authClient.DeletePlaylistItemsAsync(itemsToRemove); } var reorderedList = await authClient.GetAllPlaylistItemsAsync(playlistSynchronization.ToPlaylistId); foreach (var track in sourceSynchronizationTracks) { var desiredPosition = sourceSynchronizationTracks.IndexOf(track); var trackInReorderedList = reorderedList.Single(t => t.snippet.resourceId.videoId == track.TrackId); var existingPosition = reorderedList.IndexOf(trackInReorderedList); if (existingPosition != desiredPosition) { _logger.Debug("Moving track {0} from {1} to {2}", track.TrackId, existingPosition, desiredPosition); trackInReorderedList.snippet.position = desiredPosition; await authClient.UpdatePlaylistItemAsync(trackInReorderedList); reorderedList = Move(reorderedList, existingPosition, desiredPosition); } } await SetPlaylistInfo(sourcePlaylist, sourceSynchronizationTracks, playlistSynchronization, authClient); return new SyncCounts(videosToAdd.Count, duplicateItemsToRemove.Count, itemsToRemove.Count); } private async Task SetPlaylistInfo(GenericPlaylist sourcePlaylist, List sourceSynchronizationTracks, Data.PlaylistSynchronization playlistSynchronization, AuthenticatedYoutubeApi authClient) { const int maxCharsInDescription = 5000; var playlist = await authClient.GetPlaylistAsync(playlistSynchronization.ToPlaylistId); var tracksInPlaylist = sourcePlaylist.Tracks.Where(track => sourceSynchronizationTracks.Any(syncTrack => track.ISRC == syncTrack.ISRC)); var description = playlist.snippet.description; if (playlistSynchronization.DescriptionCopyMode == FieldCopyMode.CopySource) { description = sourcePlaylist.Description; } else if (playlistSynchronization.DescriptionCopyMode == FieldCopyMode.UseSetting) { description = playlistSynchronization.Description; } if (playlistSynchronization.AppendTrackList) { // Adds two empty rows before song list, if a description text is set. description = description + (string.IsNullOrWhiteSpace(description) ? string.Empty : Environment.NewLine + Environment.NewLine); foreach ( var trackTitle in tracksInPlaylist.Select(t => string.Format("{0} - {1}", string.Join(", ", t.Artists), t.Name))) { if (description.Length + trackTitle.Length + Environment.NewLine.Length > maxCharsInDescription) break; description += trackTitle + Environment.NewLine; } } description = StripDescriptionLinks(description); playlist.snippet.description = description; var title = string.Empty; if (playlistSynchronization.TitleCopyMode == FieldCopyMode.CopySource) { title = playlist.snippet.title; } else if (playlistSynchronization.TitleCopyMode == FieldCopyMode.UseSetting) { title = playlistSynchronization.Title; } if (!string.IsNullOrWhiteSpace(title)) { playlist.snippet.title = title; } await authClient.UpdatePlaylistAsync(playlist); } public List Move(List list, int oldIndex, int newIndex) { var item = list[oldIndex]; list.RemoveAt(oldIndex); newIndex = Math.Max(list.Count, newIndex); list.Insert(newIndex, item); return list; } private List GetTrackDuplicatesToRemove(List allPlaylistTrackItems, List sourceSynchronizationTracks) { var itemsToRemove = new List(); var trackLookup = sourceSynchronizationTracks.Select(t => t.TrackId).ToHashSet(); var trackGroups = allPlaylistTrackItems.Where(i => trackLookup.Contains(i.snippet.resourceId.videoId)).GroupBy(g => g.snippet.resourceId.videoId); foreach (var duplicateTrackGroup in trackGroups.Where(c => c.Count() > 1)) { var orderedDuplicateTracks = duplicateTrackGroup.OrderByDescending(r => r.snippet.publishedAt); itemsToRemove.AddRange(orderedDuplicateTracks.Skip(1)); } return itemsToRemove; } private List GetPlaylistItemsToRemove(List currentItems, List sourceSynchronizationTracks, List duplicateTracksToRemove) { return currentItems.Where(i => sourceSynchronizationTracks.All(t => t.TrackId != i.snippet.resourceId.videoId)).Where(i => duplicateTracksToRemove.All(dt => dt.snippet.resourceId.videoId != i.snippet.resourceId.videoId)) .ToList(); } private List GetVideosToAdd(List currentItems, List sourceSynchronizationTracks) { return sourceSynchronizationTracks.Select(t => t.TrackId).Except(currentItems.Select(i => i.snippet.resourceId.videoId)).ToList(); } private async Task> GetTrackMappings(List sourceTracks, AuthenticatedYoutubeApi authClient) { var previouslySynchronizedTracks = _synchronizationTrackManager.GetSynchronizedTracks(sourceTracks.Select(t => t.ISRC).ToList(), ServiceType.YouTube); previouslySynchronizedTracks = previouslySynchronizedTracks.Where(t => t.MatchDate > DateTime.UtcNow.AddDays(-3)).ToList(); IEnumerable videoLookup = new List(); if (previouslySynchronizedTracks.Any()) videoLookup = (await authClient.GetAllVideosAsync(previouslySynchronizedTracks.Select(t => t.TrackId).ToList())).Where(v=> v.status.uploadStatus != "rejected"); previouslySynchronizedTracks = previouslySynchronizedTracks.Where(t => videoLookup.Any(v => v.id == t.TrackId)).ToList(); var allSynchronizedTracks = new List(); foreach (var genericTrack in sourceTracks) { var synchronizedTrack = previouslySynchronizedTracks.FirstOrDefault(t => t.ISRC.Equals(genericTrack.ISRC, StringComparison.InvariantCultureIgnoreCase)); if (synchronizedTrack == null) { var videoId = await FindYouTubeVideoAsync(genericTrack, authClient); if (videoId != null) { // Commented away so we don't use cached video ids var syncTrack = await _synchronizationTrackManager.SaveSynchronizationTrackAsync(genericTrack.ISRC, ServiceType.YouTube, videoId); if (!allSynchronizedTracks.Any(t => t.TrackId.Equals(videoId, StringComparison.CurrentCultureIgnoreCase))) allSynchronizedTracks.Add(syncTrack); } } else { allSynchronizedTracks.Add(synchronizedTrack); } } return allSynchronizedTracks.DistinctBy(t=> t.TrackId).ToList(); } public async Task FindYouTubeVideoAsync(GenericTrack genericTrack, AuthenticatedYoutubeApi authClient) { var searchItems = await SearchVideos(genericTrack, authClient); if (searchItems != null && searchItems.Any()) { var displayNames = await authClient.GetChannelsAsync(searchItems.Select(r => r.snippet.channelId).ToList()); Dictionary displayNamesPerChannelId = new Dictionary(); if (displayNames != null && displayNames.items.Any()) { displayNamesPerChannelId = displayNames.items.DistinctBy(d => d.snippet.title).ToDictionary(d => d.id, d => d.snippet.title); } var matchedVideo = MatchVideos(searchItems, displayNamesPerChannelId, genericTrack); if (matchedVideo != null) return matchedVideo.id.videoId; } return null; } public SearchItem MatchVideos(List searchItems, Dictionary displayNamesPerChannelId, GenericTrack genericTrack) { //We want the video name to really contain the track and artist name var videosThatContainName = searchItems.Where(v => NameMatches(v.snippet.title, genericTrack)).ToList(); //We exclude videos that have certain blacklisted words in the title (when the word is not in the track title) var videosWithoutBlacklistedWords = videosThatContainName.Where(v => ContainsNoBlacklistedWords(v.snippet.title, genericTrack)).ToList(); //Videos are checked against a list of blacklisted channels, and not used if from one of those channels. var videosWithouthBlacklistedChannels = videosWithoutBlacklistedWords.Where(v => !FromBlacklistedChannel(v)).ToList(); //We prefer videos by VEVO users. var videosThatMayBeFromVevoUsers = videosWithouthBlacklistedChannels.Where( v => Contains(v.snippet.channelTitle, "vevo")).ToList(); if (videosThatMayBeFromVevoUsers.Any()) return videosThatMayBeFromVevoUsers.First(); //If the video is from a whitelisted channel, we like that. var videosFromWhitelistedChannels = videosWithouthBlacklistedChannels.Where(v => FromWhitelistedChannel(v)).ToList(); if (videosFromWhitelistedChannels.Any()) return videosFromWhitelistedChannels.First(); //We prefer users with artist name var videosThatMayBeFromArtist = videosWithouthBlacklistedChannels.Where(v => LooksOfficial(v, displayNamesPerChannelId.ContainsKey(v.snippet.channelId) ? displayNamesPerChannelId[v.snippet.channelId] : null, genericTrack)).ToList(); //If all else fails, just return first video (after the filtering above) return videosThatMayBeFromArtist.FirstOrDefault() ?? videosWithouthBlacklistedChannels.FirstOrDefault(); } private bool FromWhitelistedChannel(SearchItem searchItem) { var whiteListedChannels = _flaggedChannelsManager.GetWhitelistedChannels(); return whiteListedChannels.Any( channel => channel.Equals(searchItem.snippet.channelTitle, StringComparison.InvariantCultureIgnoreCase) || channel.Equals(searchItem.snippet.channelId, StringComparison.InvariantCultureIgnoreCase)); } private bool FromBlacklistedChannel(SearchItem searchItem) { var blacklistedChannels = _flaggedChannelsManager.GetBlacklistedChannels(); return blacklistedChannels.Any( channel => channel.Equals(searchItem.snippet.channelTitle, StringComparison.InvariantCultureIgnoreCase) || channel.Equals(searchItem.snippet.channelId, StringComparison.InvariantCultureIgnoreCase)); } private bool LooksOfficialV1(SearchItem v, GenericTrack genericTrack) { var channelVariations = new List(); var artistName = genericTrack.Artists.First(); artistName = TextUtility.RemoveAccent(artistName); artistName = TextUtility.RemoveSpecialCharacters(artistName); artistName = artistName.Replace(" ", string.Empty); channelVariations.Add(artistName); channelVariations.Add(artistName + "official"); return channelVariations.Any(channelVariation => v.snippet.channelTitle.Equals(channelVariation, StringComparison.InvariantCultureIgnoreCase)); } private bool LooksOfficial(SearchItem v, string channelDisplayName, GenericTrack genericTrack) { var normalizedArtistName = genericTrack.Artists.First(); normalizedArtistName = TextUtility.RemoveAccent(normalizedArtistName); normalizedArtistName = TextUtility.RemoveSpecialCharacters(normalizedArtistName); normalizedArtistName = normalizedArtistName.Replace(" ", string.Empty); if (!string.IsNullOrWhiteSpace(channelDisplayName)) { var normalizedChannelName = TextUtility.RemoveAccent(channelDisplayName); normalizedChannelName = TextUtility.RemoveSpecialCharacters(normalizedChannelName); normalizedChannelName = normalizedChannelName.Replace(" ", string.Empty); var displayNameVariations = new List() {normalizedArtistName}; displayNameVariations.AddRange(_officialLookingSuffixes.Select(suffix => normalizedArtistName + suffix)); var displayNameLooksOfficial = displayNameVariations.Any( displayNameVariation => displayNameVariation.Equals(normalizedChannelName, StringComparison.CurrentCultureIgnoreCase)); if (displayNameLooksOfficial) return true; } var channelVariations = new List(); channelVariations.Add(normalizedArtistName); channelVariations.Add(normalizedArtistName + "official"); return channelVariations.Any(channelVariation => v.snippet.channelTitle.Equals(channelVariation, StringComparison.InvariantCultureIgnoreCase)); } private bool NameMatches(string title, GenericTrack genericTrack) { var trackName = GetNormalizedTrackName(genericTrack.Name); var normalizedYoutubeTitle = GetNormalizedYoutubeTitle(title); if (!ContainsFullTitle(normalizedYoutubeTitle, trackName)) return false; var firstArtist = genericTrack.Artists.First(); var artistVariations = TextUtility.GetArtistVariations(firstArtist); bool containsArtistName = artistVariations.Any(a => Contains(title, a)); if (!containsArtistName) return false; return true; } private bool ContainsNoBlacklistedWords(string title, GenericTrack genericTrack) { // Check if blacklisted words are in track title var blacklistedWordsNotInTrack = _blacklistedYouTubeTitleWords.Where(word => genericTrack.Name.IndexOf(word, StringComparison.CurrentCultureIgnoreCase) < 0).ToList(); // Check if blacklisted words are in artist names blacklistedWordsNotInTrack = blacklistedWordsNotInTrack.Where(word => !genericTrack.Artists.Any(artist => artist.IndexOf(word, StringComparison.CurrentCultureIgnoreCase) >= 0)).ToList(); if (blacklistedWordsNotInTrack.Any(word => title.IndexOf(word, StringComparison.CurrentCultureIgnoreCase) >= 0)) return false; return true; } private bool Contains(string text, string searchTerm) { return text.IndexOf(searchTerm, StringComparison.InvariantCultureIgnoreCase) >= 0; } private bool ContainsFullTitle(string text, string searchTerm) { return Contains(text, " " + searchTerm + " ") || text.StartsWith(searchTerm + " ", StringComparison.InvariantCultureIgnoreCase) || text.EndsWith(" " + searchTerm, StringComparison.InvariantCultureIgnoreCase); } private string GetNormalizedYoutubeTitle(string title) { var normalizedTitle = title; normalizedTitle = TextUtility.RemoveAccent(normalizedTitle); normalizedTitle = TextUtility.RemoveSpecialCharacters(normalizedTitle); return normalizedTitle.Trim(); } private string GetNormalizedTrackName(string title) { var normalizedTitle = title; normalizedTitle = StripExtraTrackInfo(normalizedTitle); normalizedTitle = TextUtility.RemoveAccent(normalizedTitle); normalizedTitle = TextUtility.RemoveSpecialCharacters(normalizedTitle); return normalizedTitle.Trim(); } private static string StripExtraTrackInfo(string normalizedTitle) { normalizedTitle = Regex.Replace(normalizedTitle.ToLowerInvariant(), "\\(.*?\\)", string.Empty); normalizedTitle = Regex.Replace(normalizedTitle, "-.*$", string.Empty); return normalizedTitle; } public async Task> SearchVideos(GenericTrack genericTrack, AuthenticatedYoutubeApi authClient) { var artistName = genericTrack.Artists.FirstOrDefault(); string trackNameWithoutSpecialInfo = StripExtraTrackInfo(genericTrack.Name); string specialInfo = string.Empty; var parenthesisExtraInfoMatches = Regex.Match(genericTrack.Name, "\\(.*?\\)"); if (parenthesisExtraInfoMatches.Groups.Count > 0) specialInfo = parenthesisExtraInfoMatches.Groups[0].Value.Replace("(", string.Empty).Replace(")", string.Empty).Trim(); var dashExtraInfoMatches = Regex.Match(genericTrack.Name, "-.*$"); if (dashExtraInfoMatches.Groups.Count > 0) specialInfo = dashExtraInfoMatches.Groups[0].Value.Replace("-",string.Empty).Trim(); var searchQueryNoQuotes = artistName + " - " + trackNameWithoutSpecialInfo; var searchQuery = "\"" + searchQueryNoQuotes + "\""; if (!string.IsNullOrWhiteSpace(specialInfo)) { specialInfo = StripBlacklistedInfo(specialInfo); searchQuery += " " + specialInfo; searchQueryNoQuotes += " " + specialInfo; } var searchResult = await authClient.SearchVideosAsync(searchQuery, 25, CancellationToken.None); var searchResultNoQuotes = await authClient.SearchVideosAsync(searchQueryNoQuotes, 25, CancellationToken.None); var searchItems = searchResult.items.Union(searchResultNoQuotes.items).DistinctBy(i => i.id.videoId).ToList(); return searchItems; } private string StripBlacklistedInfo(string specialInfo) { if (_blacklistedPartialPhrases.Any( phrase => specialInfo.IndexOf(phrase, StringComparison.CurrentCultureIgnoreCase) > 0)) return string.Empty; _blacklistedExactPhrases.ForEach(exactPhrase => specialInfo = specialInfo.ToLowerInvariant().Replace(exactPhrase.ToLowerInvariant(), string.Empty)); return specialInfo; } private async Task> InsertSelectedVideos(List allTracks, Data.PlaylistSynchronization playlistSync, AuthenticatedYoutubeApi authClient) { var videosToAdd = playlistSync.InsertMedia; // - 1 on position because of zero index in insert0 videosToAdd.ForEach(m => m.InsertPosition--); if (videosToAdd == null || !videosToAdd.Any()) return allTracks; var lookup = (await authClient.GetAllVideosAsync(videosToAdd.Select(v => v.MediaId).ToList())).Where(v => v.status.uploadStatus != "rejected"); videosToAdd = videosToAdd.Where(v => lookup.Any(l => l.id == v.MediaId)).ToList(); foreach (var video in videosToAdd) { var newSyncTrack = new SynchronizationTrack() { TrackId = video.MediaId }; _logger.Log(LogLevel.Info, string.Format("Inserting video with ID {0} into playlist at position {1}.", video.MediaId, video.InsertPosition + 1)); if (video.InsertPosition < 0) video.InsertPosition = 0; if (video.InsertPosition <= allTracks.Count) { allTracks.Insert(video.InsertPosition, newSyncTrack); } else { allTracks.Add(newSyncTrack); } } return allTracks; } private string StripDescriptionLinks(string description) { var originalDescription = description; try { var parser = new HtmlParser(); var document = parser.Parse(description); var links = document.GetElementsByTagName("a"); foreach(var link in links) { var linkHref = link.GetAttribute("href"); var content = link.Text(); var replacement = string.Empty; //Only tries to handle track, album, artist, user and playlist (playlist uri:s also start with "spotify:user:") as spotify uris if (!string.IsNullOrWhiteSpace(linkHref) && (linkHref.StartsWith("spotify:user:") || linkHref.StartsWith("spotify:artist:") || linkHref.StartsWith("spotify:album:") || linkHref.StartsWith("spotify:track:"))) { var spotifyLink = new SpotifyLink(linkHref).Url; if (!string.IsNullOrWhiteSpace(spotifyLink)) linkHref = spotifyLink; replacement = "(" + linkHref + ")"; } if (!string.IsNullOrWhiteSpace(content)) { replacement = content + (!string.IsNullOrWhiteSpace(replacement) ? " " + replacement : string.Empty); } description = description.Replace(link.OuterHtml, replacement); } } catch(Exception) { return originalDescription; } return description; } } }