using System.Collections.Concurrent; using System.Text.RegularExpressions; using MoreLinq; using Sony.Filtr.Contracts.Abstractions; using Sony.Filtr.Contracts.Entities; using Sony.Filtr.Core; using Sony.Filtr.DistributedCaching; using System; using System.Collections.Generic; using System.Linq; using Artist = Sony.Filtr.Contracts.Entities.Artist; using System.Threading.Tasks; using NLog; using Sony.Filtr.Contracts.Definitions; using Sony.Filtr.Contracts.Entities.Genre; using Sony.Filtr.Core.Blacklists; using Sony.Filtr.Core.SonyMusic; using Sony.Filtr.Utility.Extensions; namespace Sony.Filtr.PlaylistGeneration { public class TrackSuggestionManager2 { private readonly IApplicationInstanceManager _applicationInstanceManager; private readonly PlaylistGenerationManager _playlistGenerationManager; private readonly DistributedCacheHandler _distributedCacheHandler; private readonly BlacklistManager _blacklistManager; private readonly SonyMusicManager _sonyMusicManager; private readonly Logger _logger; public TrackSuggestionManager2(IApplicationInstanceManager applicationInstanceManager, PlaylistGenerationManager playlistGenerationManager, DistributedCacheHandler distributedCacheHandler, BlacklistManager blacklistManager, SonyMusicManager sonyMusicManager) { _applicationInstanceManager = applicationInstanceManager; _playlistGenerationManager = playlistGenerationManager; _distributedCacheHandler = distributedCacheHandler; _blacklistManager = blacklistManager; _sonyMusicManager = sonyMusicManager; _logger = LogManager.GetLogger("TrackSuggestionManager2"); } public async Task> GetSuggestedTracksAsync(Application application, string spotifyRegionCode, Artist artist, ServiceType serviceType = ServiceType.Spotify) { //var tracksPerArtist = SettingsManager.Instance.GetIntSetting("numberOfTracksPerArtist", application); //var maxArtists = SettingsManager.Instance.GetIntSetting("numberOfArtists", application) * 2; const int tracksPerArtist = 5; const int maxTracks = 1000; var requestId = Guid.NewGuid(); var cacheKey = "GetSuggestedTracks-3.7:" + GetCacheKey3(application, spotifyRegionCode, artist, tracksPerArtist, maxTracks, serviceType); List trackSuggestions = _distributedCacheHandler.GetList(cacheKey) as List; if (trackSuggestions != null) { _logger.Debug("Request: {0}. Found existing recommendation for {1} for region {2}", requestId, artist.Name, spotifyRegionCode); return trackSuggestions; } List allArtists = new List(); allArtists.Add(new SimilarArtistMatch(artist, 100)); allArtists.AddRange(await _playlistGenerationManager.GetSimilarArtistsAsync(artist, 100)); //TODO: If duplicate we should only keep the one with highest score var artists = allArtists.Where(a => a != null && a.Artist != null).ToList(); artists = IsNotBlackListed(application, artists).ToList(); //BoostSonyArtists2(artists); artists = artists.DistinctBy(a => a.Artist.Name).OrderByDescending(s => s.Match).ThenByDescending(s => s.Artist.IsSonyArtist).ToList(); var localSuggestions = new ConcurrentBag(); Dictionary> allArtistTracks = await GetTracksAsync(artists.Select(a=> a.Artist.Name).ToList(), spotifyRegionCode, serviceType); foreach (var artistToUse in artists) { SimilarArtistMatch localArtistToUse = artistToUse; var key = localArtistToUse.Artist.Name; if (allArtistTracks.ContainsKey(key)) { List artistTracks = allArtistTracks[key]; artistTracks = FilterTracks(artistTracks, serviceType); artistTracks = artistTracks.OrderByDescending(t => t.IsSonyTrack).ThenBy(t => t.Index).ToList(); if (artistTracks.Any()) { var tracksToAdd = artistTracks.Take(tracksPerArtist).ToList(); if (tracksToAdd.Any()) { var artistIsSony = artistTracks.Any(t => t.IsSonyTrack); tracksToAdd.Select( t => new ExtendedTrackSuggestion() { Artist = localArtistToUse.Artist, ExtendedTrack = t.ExtendedTrack, Score = artistIsSony ? (int)(localArtistToUse.Match * 1.5) : (int)localArtistToUse.Match }) .ForEach(localSuggestions.Add); } } } } _logger.Debug("Request {0}. Done with artist {1} and region {2}", requestId, artist.Name, spotifyRegionCode); if (serviceType == ServiceType.Spotify) trackSuggestions = localSuggestions.DistinctBy(t => t.ExtendedTrack.SpotifyLink).OrderByDescending(t => t.Score).Take(maxTracks).ToList(); else if (serviceType == ServiceType.Deezer) trackSuggestions = localSuggestions.DistinctBy(t => t.ExtendedTrack.DeezerId).OrderByDescending(t => t.Score).Take(maxTracks).ToList(); _distributedCacheHandler.PutList(cacheKey, trackSuggestions, DateTime.UtcNow.AddDays(2)); return trackSuggestions; } private List FilterTracks(List artistTracks, ServiceType? serviceType) { if (serviceType.HasValue && serviceType.Value == ServiceType.Deezer) { return artistTracks.Where(t => t.ExtendedTrack.DeezerId.HasValue).ToList(); } return artistTracks; } private async Task>> GetTracksAsync(List artists, string spotifyRegionCode, ServiceType serviceType) { var allArtistsTrackInfos = new ConcurrentDictionary>(); var allArtistTracks = await _playlistGenerationManager.GetExtendedTopTracksForArtistsAsync(artists.ToList(), 100, spotifyRegionCode, serviceType: serviceType); //var allTrackISRC = allArtistTracks.SelectMany(t => t.Value).Select(t => t.ISRC).ToList(); var sonyLookup = await GetSonyTrackLookup(spotifyRegionCode, allArtistTracks); foreach (var artistTracks in allArtistTracks) { var trackInfos = artistTracks.Value.ItemIndex().Select(t => new TrackInfo() { ExtendedTrack = t.Item, IsSonyTrack = sonyLookup.Contains(t.Item.ISRC), Index = t.Index }).ToList(); allArtistsTrackInfos.TryAdd(artistTracks.Key, trackInfos); } return allArtistsTrackInfos.ToDictionary(k=> k.Key, v=> v.Value); } private async Task> GetSonyTrackLookup(string spotifyRegionCode, Dictionary> allArtistTracks) { ConcurrentBag sonyISRCs = new ConcurrentBag(); var sonyLookupArtistTracks = new ConcurrentDictionary>(); foreach (var artistTracks in allArtistTracks) { var sonyTracksCacheKey = "SonyTracks:" + artistTracks.Key + ":" + spotifyRegionCode; IEnumerable sonyTracks = _distributedCacheHandler.Get(sonyTracksCacheKey) as List; if (sonyTracks != null) { sonyTracks.ForEach(t => sonyISRCs.Add(t)); } else { sonyLookupArtistTracks.TryAdd(artistTracks.Key, artistTracks.Value); } } var sonyLookupArtistTracksIsrc = sonyLookupArtistTracks.SelectMany(p => p.Value).Select(t => t.ISRC).Where(i=> !string.IsNullOrWhiteSpace(i)).Distinct().ToList(); _logger.Debug("Doing sony track lookup for tracks from {0} artists in region {1}", sonyLookupArtistTracks.Count, spotifyRegionCode); ConcurrentBag extraLookupSonyTracks = new ConcurrentBag(); foreach (var isrcBatch in sonyLookupArtistTracksIsrc.Batch(1000)) { var sonyIsrcs = await _sonyMusicManager.IsSonyTracksAsync(isrcBatch.ToList(), spotifyRegionCode); sonyIsrcs.ForEach(t => extraLookupSonyTracks.Add(t)); } foreach (var artistTracks in sonyLookupArtistTracks) { var artistISRCs = artistTracks.Value.Where(t => !string.IsNullOrWhiteSpace(t.ISRC)).Select(t => t.ISRC); var artistSonyISRCs = extraLookupSonyTracks.Intersect(artistISRCs).ToList(); var sonyTracksCacheKey = "SonyTracks:" + artistTracks.Key + ":" + spotifyRegionCode; _distributedCacheHandler.Put(sonyTracksCacheKey, artistSonyISRCs, DateTime.Now.AddDays(3)); artistSonyISRCs.ForEach(a => sonyISRCs.Add(a)); } var sonyLookup = sonyISRCs.ToLookup(t => t); return sonyLookup; } public async Task> GetTracksAsync(Artist artist, string spotifyRegionCode) { var artistTracks = (await _playlistGenerationManager.GetExtendedTopTracksAsync(artist, 100, spotifyRegionCode)).ToList(); var sonyTracksCacheKey = "SonyTracks:" + artist.Name + ":" + spotifyRegionCode; IEnumerable sonyTracks = _distributedCacheHandler.Get(sonyTracksCacheKey) as List; if (sonyTracks == null) { sonyTracks = await _sonyMusicManager.IsSonyTracksAsync(artistTracks.Where(t => !string.IsNullOrWhiteSpace(t.ISRC)).Select(t => t.ISRC).ToList(), spotifyRegionCode); _distributedCacheHandler.Put(sonyTracksCacheKey, sonyTracks, DateTime.Now.AddDays(1)); } var sonyLookup = sonyTracks.ToLookup(t => t); return artistTracks.ItemIndex().Select(t => new TrackInfo() { ExtendedTrack = t.Item, IsSonyTrack = sonyLookup.Contains(t.Item.ISRC), Index = t.Index }).ToList(); } public async Task> GetSuggestedTracksForGenreNewAsync(Genre genre, string regionCode, SearchTracksParameters searchTracksParameters = null) { var cacheKey = GetSuggestedTracksForGenreCacheKey(genre.LastFMTag, regionCode, null, searchTracksParameters); var trackSuggestions = _distributedCacheHandler.GetList(cacheKey) as List; if (trackSuggestions != null) { //Console.WriteLine("Using cached list"); return trackSuggestions; } const int maxTracks = 1000; var localTrackSuggestions = new ConcurrentBag(); var echoTracks = await _playlistGenerationManager.GetExtendedTracksForTag(genre, regionCode); if (searchTracksParameters != null) { echoTracks = echoTracks.Where(p => MatchesParameters(p.Item, searchTracksParameters)).ToList(); } foreach (var artistTrackGroup in echoTracks.GroupBy(t=> t.Item.ArtistName)) { var bestTrackRank = artistTrackGroup.Min(a => a.Weight); double matchValue = Math.Max(maxTracks - bestTrackRank, 1); foreach (var track in artistTrackGroup) { var suggestion = new ExtendedTrackSuggestion() { Artist = new Artist(track.Item.ArtistName), ExtendedTrack = track.Item, Score = (int)matchValue }; localTrackSuggestions.Add(suggestion); } } var tracks = localTrackSuggestions.DistinctBy(t => t.ExtendedTrack.SpotifyLink).OrderByDescending(t => t.Score).Take(maxTracks).ToList(); //Caching disabled for now //DistributedCacheHandler.Instance.PutList(cacheKey, tracks, DateTime.UtcNow.AddDays(2)); return tracks; } public async Task> GetSuggestedTracksForGenreAsync(Genre genre, string regionCode, ServiceType serviceType = ServiceType.Spotify, SearchTracksParameters searchTracksParameters = null) { var cacheKey = GetSuggestedTracksForGenreCacheKey(genre.LastFMTag, regionCode, serviceType, searchTracksParameters); var trackSuggestions = _distributedCacheHandler.GetList(cacheKey) as List; if (trackSuggestions != null && trackSuggestions.Any()) { //Console.WriteLine("Using cached list"); return trackSuggestions; } const int maxTracks = 300; const int tracksPerArtist = 3; var localTrackSuggestions = new ConcurrentBag(); var topTracks = await _playlistGenerationManager.GetTopTracksForTagAsync(genre.LastFMTag); Console.WriteLine("Found {0} top tracks on lastfm", topTracks.Count); var artistsNames = topTracks.Select(t => t.Item.ArtistName.Trim()).Distinct().ToList(); var artistTracks = await GetTracksAsync(artistsNames, regionCode, serviceType); var artists = await _playlistGenerationManager.GetArtistsAsync(artistsNames); foreach (var artistGroup in topTracks.GroupBy(g => g.Item.ArtistName)) { var artistName = artistGroup.Key; var artist = artists.FirstOrDefault(a => a.Name.Equals(artistName, StringComparison.InvariantCultureIgnoreCase)); if (artist == null || !artistTracks.ContainsKey(artistName)) { Console.WriteLine("Couldn't find artist {0}", artistName); continue; } var lastFmTracks = artistGroup.Select(t => t).ToList(); var localTracks1 = artistTracks[artistName]; localTracks1 = FilterTracks(localTracks1, serviceType); if (localTracks1 == null) { Console.WriteLine("Found no localTracks for artist {0}", artistName); continue; } //Console.WriteLine("Found {0} localTracks for artist {1}", localTracks1.Count(), artistName); var localTracks = localTracks1.ToList(); localTracks = localTracks.Join(lastFmTracks, localTrack => GetNormalizedTrackName(localTrack.ExtendedTrack.Title), lastFmTrack => GetNormalizedTrackName(lastFmTrack.Item.Name), (localTrack, lastFmTrack) => new { LastFmTrack = lastFmTrack, LocalTrack = localTrack }) .OrderByDescending(l => l.LocalTrack.IsSonyTrack).ThenBy(l => l.LastFmTrack.Weight) .Select(l => l.LocalTrack).DistinctBy(t => t.ExtendedTrack.Title).Take(tracksPerArtist).ToList(); //Console.WriteLine("Found {0} joined tracks for artist {1}", localTracks.Count(), artistName); if (searchTracksParameters != null) { localTracks = FilterTracks(localTracks, searchTracksParameters).ToList(); } //Console.WriteLine("Found {0} joined tracks after filtering for artist {1}", localTracks.Count(), artistName); if (localTracks.Any()) { var bestTrackRank = artistGroup.Min(a => a.Weight); double matchValue = Math.Max(maxTracks - bestTrackRank, 1); if (localTracks.First().IsSonyTrack) matchValue = matchValue * 2; else if (artist.IsSonyArtist) matchValue = matchValue * 1.2; foreach (var track in localTracks.Select(t => new ExtendedTrackSuggestion() { Artist = artist, ExtendedTrack = t.ExtendedTrack, Score = (int)matchValue })) { //Console.WriteLine("Added track {0} for artist {1}", track.ExtendedTrack.Title, artistName); localTrackSuggestions.Add(track); } } else { //if (localTracks1.Any()) //{ // foreach (var lastFmTrack in lastFmTracks) // { // Console.WriteLine("Artist: {0}. LastFM-Track: {1}. Did not match any of {2}", artistName, lastFmTrack.Item.Name, string.Join("| ", localTracks1.Select(t => t.Title))); // } //} } } Console.WriteLine("Got {0} track suggestions ", localTrackSuggestions.Count); List tracks = new List(); if (serviceType == ServiceType.Spotify) tracks = localTrackSuggestions.DistinctBy(t=> t.ExtendedTrack.SpotifyLink).OrderByDescending(t=> t.Score).Take(maxTracks).ToList(); else if (serviceType == ServiceType.Deezer) tracks = localTrackSuggestions.DistinctBy(t=> t.ExtendedTrack.DeezerId).OrderByDescending(t=> t.Score).Take(maxTracks).ToList(); _distributedCacheHandler.PutList(cacheKey, tracks, DateTime.UtcNow.AddDays(2)); return tracks; } private string GetNormalizedTrackName(string title) { var normalizedTitle = Regex.Replace(title.ToLowerInvariant(), "\\(.*?\\)", string.Empty); normalizedTitle = Regex.Replace(normalizedTitle, "-.*$", string.Empty); normalizedTitle = normalizedTitle.Replace(" & ", " and "); normalizedTitle = normalizedTitle.Replace("'", string.Empty); normalizedTitle = normalizedTitle.Replace("`", string.Empty); normalizedTitle = normalizedTitle.Replace("´", string.Empty); normalizedTitle = normalizedTitle.Replace("!", string.Empty); normalizedTitle = normalizedTitle.Replace(".", string.Empty); normalizedTitle = normalizedTitle.Replace("?", string.Empty); normalizedTitle = normalizedTitle.Replace("\"", string.Empty); return normalizedTitle.Trim(); } private string GetSuggestedTracksForGenreCacheKey(string genre, string regionCode, ServiceType? serviceType, SearchTracksParameters searchTracksParameters) { var cacheKey = "GetSuggestedTracksForGenre-2.6:" + typeof(ExtendedTrack) + ":" + genre + " " + regionCode; var serviceTypeString = (serviceType.HasValue) ? serviceType.Value.ToString() : "null"; cacheKey += "-" + serviceTypeString; if (searchTracksParameters != null) cacheKey += "|" + GetSearchTracksParameterCacheKey(searchTracksParameters); return cacheKey; } private void BoostSonyArtists2(IEnumerable artistsToUse) { artistsToUse.Where(a => a.Artist.IsSonyArtist).ToList().ForEach(a => a.Match = BoostSonyMatchValue2(a.Match)); } private double BoostSonyMatchValue2(double score) { return score*1.5; } private IEnumerable IsNotBlackListed(Application app, IEnumerable artistsToUse) { var blacklistedArtists = _blacklistManager.GetBlacklistedArtists(app); var globalBlacklistedArtists = _blacklistManager.GetBlacklistedArtists(_applicationInstanceManager.GetGlobalPushApplication()); var lookup = blacklistedArtists.Union(globalBlacklistedArtists).ToLookup(b=> b.ArtistName.ToLowerInvariant()); return artistsToUse.Where(a => !lookup.Contains(a.Artist.Name.ToLowerInvariant())); } private string GetCacheKey3(Application application, string spotifyRegionCode, Artist artist, int tracksPerArtist, int maxArtists, ServiceType? serviceType) { if (spotifyRegionCode == null) spotifyRegionCode = "null"; var serviceTypeString = (serviceType.HasValue) ? serviceType.Value.ToString() : "null"; return string.Join("-", "GetSuggestedTracks-v6-", typeof(ExtendedTrack), application.ID, spotifyRegionCode.ToLower(), artist.Name, tracksPerArtist, maxArtists, serviceTypeString); } public async Task> GetSuggestedTracksWithLocalFilteringAsync(Application application, string spotifyRegionCode, string artistName, SearchTracksParameters searchTracksParameters) { var artist = new Artist(artistName); const int tracksPerArtist = 5; const int maxTracks = 1000; List allArtists = new List(); allArtists.Add(new SimilarArtistMatch(artist, 100)); allArtists.AddRange(await _playlistGenerationManager.GetSimilarArtistsAsync(artist, 100)); //TODO: If duplicate we should only keep the one with highest score var artists = allArtists.Where(a => a != null && a.Artist != null).ToList(); artists = IsNotBlackListed(application, artists).ToList(); BoostSonyArtists2(artists); artists = artists.DistinctBy(a => a.Artist.Name).OrderByDescending(s => s.Match).ThenByDescending(s => s.Artist.IsSonyArtist).ToList(); var localSuggestions = new ConcurrentBag(); await artists.ForEachAsync(5, async artistToUse => { SimilarArtistMatch localArtistToUse = artistToUse; IEnumerable artistTracks = await GetTracksAsync(artistToUse.Artist, spotifyRegionCode); if (artistTracks != null) { var filteredTracks = FilterTracks(artistTracks, searchTracksParameters); //Prototype with local filtering! var tracksToAdd = filteredTracks.Take(tracksPerArtist).ToList(); if (tracksToAdd.Any()) { tracksToAdd.Select(t => new ExtendedTrackSuggestion() { Artist = localArtistToUse.Artist, ExtendedTrack = t.ExtendedTrack, Score = (int)localArtistToUse.Match }) .ForEach(localSuggestions.Add); } } }); return localSuggestions.DistinctBy(t => t.ExtendedTrack.SpotifyLink).OrderByDescending(t => t.Score).ThenByDescending(t=> t.Artist.Name).ThenByDescending(t=> t.ExtendedTrack.Title).Take(maxTracks).ToList(); } private string GetSearchTracksParameterCacheKey(SearchTracksParameters searchTracksParameters) { return "SearchTracksParameters-1.2:" + string.Join("|", "First", searchTracksParameters.Tag, searchTracksParameters.Artists, searchTracksParameters.MaxDanceability, searchTracksParameters.MinDanceability, searchTracksParameters.MaxTempo, searchTracksParameters.MinTempo, searchTracksParameters.MaxEnergy, searchTracksParameters.MinEnergy, searchTracksParameters.MaxHotness, searchTracksParameters.MinHotness, searchTracksParameters.MaxDuration, searchTracksParameters.MinDuration); } private IEnumerable FilterTracks(IEnumerable artistTracks, SearchTracksParameters searchTracksParameters) { return artistTracks.Where(t => MatchesParameters(t.ExtendedTrack, searchTracksParameters)); } private bool MatchesParameters(ExtendedTrack track, SearchTracksParameters searchTracksParameters) { return IsWithinRangeAndSet(track.Tempo, searchTracksParameters.MinTempo, searchTracksParameters.MaxTempo) && IsWithinRangeAndSet(track.Danceability, searchTracksParameters.MinDanceability, searchTracksParameters.MaxDanceability) && IsWithinRangeAndSet(track.Energy, searchTracksParameters.MinEnergy, searchTracksParameters.MaxEnergy) && IsWithinRangeAndSet(track.Hotness, searchTracksParameters.MinHotness, searchTracksParameters.MaxHotness) && IsWithinRangeAndSet(track.Duration, searchTracksParameters.MinDuration, searchTracksParameters.MaxDuration); } private bool IsWithinRangeAndSet(int value, int? min, int? max) { if ((min.HasValue || max.HasValue) && value == 0) return false; return (!min.HasValue || value >= min) && (!max.HasValue || value <= max); } private bool IsWithinRangeAndSet(double? value, double? min, double? max) { if (value.HasValue) { return (!min.HasValue || value >= min) && (!max.HasValue || value <= max); } return false; } private bool IsWithinRangeOrNull(double? value, double? min, double? max) { if (!value.HasValue) return true; if (min.HasValue || max.HasValue) { return (!min.HasValue || value >= min) && (!max.HasValue || value <= max); } return true; } } public class TrackInfo { public ExtendedTrack ExtendedTrack { get; set; } public bool IsSonyTrack { get; set; } public int Index { get; set; } } public class SearchTracksParameters { public string[] Artists { get; set; } public double? MinTempo { get; set; } public double? MaxTempo { get; set; } public double? MinDanceability { get; set; } public double? MaxDanceability { get; set; } public double? MinEnergy { get; set; } public double? MaxEnergy { get; set; } public double? MinHotness { get; set; } public double? MaxHotness { get; set; } public int? MaxDuration { get; set; } public int? MinDuration { get; set; } public string Tag { get; set; } } }