using System; using System.Collections.Concurrent; using System.Collections.Generic; using System.Linq; using System.Threading; using System.Threading.Tasks; using Flurl.Util; using MoreLinq; using NLog; using Sony.Filtr.Contracts.Definitions; using Sony.Filtr.Contracts.Entities; using Sony.Filtr.Contracts.Entities.Genre; using Sony.Filtr.Core.Genres; using Sony.Filtr.DeezerAPI; using Sony.Filtr.DeezerAPI.Model; using Sony.Filtr.DistributedCaching; using Sony.Filtr.ErrorLogging; using Sony.Filtr.PlaylistGeneration.Caching; using Sony.Filtr.PlaylistGeneration.Data; using Sony.Filtr.PlaylistGeneration.MusicInfoStore; using Sony.Filtr.SpotifyWebAPI; using Sony.Filtr.SpotifyWebAPI.Model; using Sony.Filtr.Utility; using Sony.Filtr.Utility.Extensions; using StackExchange.Redis; using Artist = Sony.Filtr.Contracts.Entities.Artist; using Track = Sony.Filtr.Contracts.Entities.Track; namespace Sony.Filtr.PlaylistGeneration { public class PlaylistGenerationManager { private readonly LastFmManager _lastFmMusicInfoStore; private readonly ArtistCorrectionManager _artistCorrectionManager; private static SpotifyWebApi _spotifyWebApi; private readonly DistributedCacheHandler _distributedCacheHandler; private readonly ICacheProvider _cacheProvider; private readonly DeezerApi _deezerApi; private readonly GenreManager _genreManager; private readonly IDatabase _redisDatabase; private readonly Logger _logger; public PlaylistGenerationManager(LastFmManager lastFmManager, ArtistCorrectionManager artistCorrectionManager, SpotifyWebApi spotifyWebApi, DistributedCacheHandler distributedCacheHandler, MultiplexCacheProvider multiplexCacheProvider, DeezerApi deezerApi, GenreManager genreManager, IDatabase redisDatabase) { _lastFmMusicInfoStore = lastFmManager; _artistCorrectionManager = artistCorrectionManager; _spotifyWebApi = spotifyWebApi; _distributedCacheHandler = distributedCacheHandler; _cacheProvider = multiplexCacheProvider; _deezerApi = deezerApi; _genreManager = genreManager; _redisDatabase = redisDatabase; _logger = LogManager.GetLogger("TrackSuggestionManager2"); } public async Task> GetArtistsAsync(List artistNames) { if (artistNames == null || !artistNames.Any()) return new List(); //Check cache with search term var artists = await _cacheProvider.FetchArtistsFromCacheAsync(artistNames); var missingArtists = artistNames.Except(artists.Select(a => a.Name)).ToList(); if (missingArtists.Any()) { FetchMissingArtistsInBackground(missingArtists); } return artists; } private void FetchMissingArtistsInBackground(IEnumerable missingArtists) { ThreadPool.QueueUserWorkItem(async ms => { try { foreach (var missingArtist in missingArtists) { var artist = await _lastFmMusicInfoStore.GetArtistAsync(missingArtist); if (artist == null) { _artistCorrectionManager.SetKnownUnknown(missingArtist); continue; } if (!artist.Name.Equals(missingArtist, StringComparison.InvariantCultureIgnoreCase)) { _artistCorrectionManager.SetMisspelled(missingArtist, artist.Name); } _cacheProvider.CacheArtist(artist); } } catch (Exception ex) { ErrorLoggingManager.Instance.LogError(ex); } }); } public async Task GetArtistAsync(string artistName) { if (string.IsNullOrWhiteSpace(artistName)) return null; Artist artist = null; //Check cache with search term artist = _cacheProvider.FetchArtistFromCache(artistName); if (artist != null) return artist; //Pre-Cache-check. Known misspelling or known missing at last.fm? if (_artistCorrectionManager.IsKnownUnknown(artistName)) return null; var corrected = _artistCorrectionManager.GetCorrectSpelling(artistName); if (!string.IsNullOrWhiteSpace(corrected)) { artistName = corrected; //Check cache with search term artist = _cacheProvider.FetchArtistFromCache(artistName); if (artist != null) return artist; } artist = await _lastFmMusicInfoStore.GetArtistAsync(artistName); if (artist == null) { _artistCorrectionManager.SetKnownUnknown(artistName); return null; } if (!artist.Name.Equals(artistName, StringComparison.InvariantCultureIgnoreCase)) { _artistCorrectionManager.SetMisspelled(artistName, artist.Name); //Check cache with artist name found on last.fm artist = _cacheProvider.FetchArtistFromCache(artistName); if (artist != null) return artist; } if (artist != null) _cacheProvider.CacheArtist(artist); return artist; } public async Task> GetGenresAsync(Application application, Artist artist) { List genreMatches = new List(); var tags = await _cacheProvider.FetchArtistTagsFromCacheAsync(artist.Name); if (tags == null) { tags = await _lastFmMusicInfoStore.GetTagsAsync(artist); if (tags != null && tags.Any()) _cacheProvider.CacheArtistTags(artist.Name, tags); else return genreMatches; } genreMatches = _genreManager.MatchGenres(application, tags).ToList(); return genreMatches; } //New and improved public async Task>> GetArtistTagsAsync(Application application, Artist artist) { var tags = await GetArtistTagsInternalAsync(artist); var tagMatches = _genreManager.MatchTags(application, tags); return tagMatches; } private async Task> GetArtistTagsInternalAsync(Artist artist) { var tags = await _cacheProvider.FetchArtistTagsFromCacheAsync(artist.Name); if (tags == null) { tags = await _lastFmMusicInfoStore.GetTagsAsync(artist); if (tags != null) _cacheProvider.CacheArtistTags(artist.Name, tags); } return tags; } private async Task>> GetArtistTagsInternalAsync(List artists) { var artistNames = artists.Select(a => a.Name).ToList(); var artistTags = await _cacheProvider.FetchArtistsTagsFromCacheAsync(artistNames); var missingArtists = artistNames.Except(artistTags.Keys, StringComparer.InvariantCultureIgnoreCase).ToList(); if (missingArtists.Any()) { GetMissingArtistsTagsInBackground(missingArtists); } return artistTags; } private void GetMissingArtistsTagsInBackground(List missingArtists) { ThreadPool.QueueUserWorkItem(async state => { try { foreach (var missingArtist in missingArtists) { var cacheKey = "NoTags:" + missingArtist; var noTagsForArtist = _distributedCacheHandler.Get(cacheKey) as bool?; if (!noTagsForArtist.HasValue || noTagsForArtist.Value == false) { var artist = missingArtist; var tags = await _lastFmMusicInfoStore.GetTagsAsync(new Artist(artist)); if (tags != null && tags.Any()) { _cacheProvider.CacheArtistTags(artist, tags); } else { _distributedCacheHandler.Put(cacheKey, true); } } } } catch (Exception ex) { ErrorLoggingManager.Instance.LogError(ex); } }); } public async Task>>> GetArtistTagsAsync(Application app, List artists) { var allArtistTags = new ConcurrentDictionary>>(); var allGenres = _genreManager.GetGenres(app).ToList(); var tags = await GetArtistTagsInternalAsync(artists); Parallel.ForEach(artists, artist => { List artistTags; if (tags.TryGetValue(artist.Name, out artistTags)) { var localGenres = _genreManager.MatchTags(allGenres, artistTags); allArtistTags.TryAdd(artist.Name, localGenres); } }); return allArtistTags.ToDictionary(k=> k.Key, v=> v.Value); } public async Task> GetGenresAsync(Application app, Track track, bool ignoreCache = false) { var genreMatches = new List(); var tags = await _cacheProvider.FetchTrackTagsFromCacheAsync(track); if (tags == null || ignoreCache) { tags = await _lastFmMusicInfoStore.GetTagsAsync(track); if (tags == null || !tags.Any()) { return genreMatches; } await _cacheProvider.CacheTrackTagsAsync(track, tags); } genreMatches = _genreManager.MatchGenres(app, tags).ToList(); return genreMatches; } public async Task> GetSimilarArtistsAsync(Artist artist, int pageSize, bool ignoreCache = false) { //Check cache var artists = _cacheProvider.FetchSimilarArtistsFromCache(artist.Name); if (artists == null || !artists.Any() || ignoreCache) { artists = new List(); //Get from last.fm List similarArtists = await _lastFmMusicInfoStore.GetSimilarArtistsAsync(artist.Name); if (similarArtists != null && similarArtists.Any()) { foreach (var similarArtistMatch in similarArtists) { //We fetch from cache so we get missing values var cachedArtist = await GetArtistAsync(similarArtistMatch.Artist.Name); if (cachedArtist != null) similarArtistMatch.Artist = cachedArtist; } _cacheProvider.CacheSimilarArtists(artist, similarArtists); artists = similarArtists; } } return artists.Take(pageSize); } internal async Task> GetGroupedExtendedTrackAsync(List> lastFmTopTracks) { ConcurrentBag matchedTrackIds = new ConcurrentBag(); ConcurrentDictionary> mappingTable = new ConcurrentDictionary>(); lastFmTopTracks = lastFmTopTracks.Where(l => !string.IsNullOrWhiteSpace(l.Item.MBID)).ToList(); //We don't really need MBID for matching anymore but it is used in db. await lastFmTopTracks.ForEachAsync(1, async trackItem => { var track = trackItem.Item; var spotifyTrackResult = await _spotifyWebApi.SearchTracksAsync("artist:" + track.ArtistName + " " + track.Name); var matchingResult = spotifyTrackResult.tracks.items.FirstOrDefault(t => t.artists.Select(a=> a.name).Contains(track.ArtistName) && t.name.Equals(track.Name, StringComparison.InvariantCultureIgnoreCase)); if (matchingResult != null) { var trackId = matchingResult.id; mappingTable.TryAdd(trackId, trackItem); matchedTrackIds.Add(trackId); } else { } }); var extendedTracks = await FillSpotifyExtendedValuesAsync(matchedTrackIds.ToList()); var joinedItems = extendedTracks.Join(mappingTable, extendedTrack => extendedTrack.SpotifyLink.ExtractTrackID(), pair => pair.Key, (track, pair) => new GroupedExtendedTrack() { ExtendedTrack = track, MBID = pair.Value.Item.MBID, SpotifyTracks = new List() { new SpotifyWebAPI.Model.Track() { uri = track.SpotifyLink.Uri, name = track.Title, available_markets = track.Territories, artists = new List() {new SpotifyWebAPI.Model.Artist() {name = track.ArtistName } }, duration_ms = track.Duration*1000 } }, Track = pair.Value, }).ToList(); return joinedItems; } public async Task>> GetExtendedTopTracksForArtistsAsync(List artists, int artistPageSize, string spotifyRegionCode = null, bool ignoreCache = false, ServiceType serviceType = ServiceType.Spotify) { ConcurrentDictionary> allResults = new ConcurrentDictionary>(); var artistResults = await _cacheProvider.FetchExtendedTopTracksForArtistsAsync(artists, serviceType); artistResults.ForEach(a=> allResults.TryAdd(a.Key, a.Value)); var missingArtists = artists.Except(artistResults.Keys, StringComparer.InvariantCultureIgnoreCase).ToList(); if (missingArtists.Any()) { _logger.Debug("Could not find artists in local cache. Doing lookup in background for {0} artists. Some are: {1}", missingArtists.Count, string.Join(",", missingArtists.Take(5))); GetMissingArtistsInBackground(missingArtists, serviceType); // Fire-and-Forget: Will update cache in background so values are used next time. } Dictionary> filteredArtistTracks = new Dictionary>(); foreach (var artistResult in allResults) { var filteredTracks = artistResult.Value.Where(t => IsInTerritory(t, spotifyRegionCode)).Take(artistPageSize).ToList(); filteredArtistTracks.Add(artistResult.Key, filteredTracks); } return filteredArtistTracks; } private void GetMissingArtistsInBackground(List missingArtists, ServiceType serviceType) { ThreadPool.QueueUserWorkItem(delegate { foreach (var missingArtist in missingArtists) { try { var artistTracks = GetExtendedTrackFromSourceAsync(missingArtist, serviceType).Result; } catch (Exception ex) { ErrorLoggingManager.Instance.LogError(ex); } } }); } public async Task> GetExtendedTopTracksAsync(Artist artist, int pageSize, string spotifyRegionCode = null, bool ignoreCache = false, ServiceType serviceType = ServiceType.Spotify) { var artistName = artist.Name; List extendedTracks = await _cacheProvider.FetchExtendedTopTracksForArtistAsync(artistName, serviceType); if (extendedTracks == null || ignoreCache) { extendedTracks = await GetExtendedTrackFromSourceAsync(artistName, serviceType); } if (extendedTracks != null) return extendedTracks.Where(t => IsInTerritory(t, spotifyRegionCode)).Take(pageSize); return new List(); } public async Task> GetExtendedTrackFromSourceAsync(string artistName, ServiceType serviceType) { var artistHasNoTracks = _distributedCacheHandler.Get("NoTracks:" + artistName) as bool?; if (artistHasNoTracks != null && artistHasNoTracks.Value) return new List(); _logger.Debug("Lookup for artist {0}", artistName); var extendedTracks = new List(); if (serviceType == ServiceType.Spotify) { var spotifyRawTracks = await _spotifyWebApi.GetTopTracksAsync(artistName, 100); //spotifyRawTracks = spotifyRawTracks.Where(t => IsSpecifiedArtist(t, artistName)).ToList(); var spotifyTracks = BuildSpotifyTracks(spotifyRawTracks); if (spotifyTracks != null && spotifyTracks.Any()) { extendedTracks = await GetExtendedTracksAsync(spotifyTracks); } } else if (serviceType == ServiceType.Deezer) { var desiredArtistTracks = 100; //var artistSearchQuery = string.Format("artist:'{0}'", artistName); _logger.Debug("Searching Deezer for artist {0}", artistName); var artistSearch = await _deezerApi.SearchArtistsAsync(artistName, limit: 5); if (artistSearch.data.Count > 0) { var bestArtistMatch = artistSearch.data.FirstOrDefault(a => a.type == "artist" && TextUtility.LooksLikeSpecifiedArtist(a.name, artistName)); if (bestArtistMatch != null) { _logger.Debug("Got good artist search result from Deezer: {0}", bestArtistMatch.name); var artistTracksResponse = await _deezerApi.GetArtistTopTracksAsync(bestArtistMatch.id, limit: desiredArtistTracks); _logger.Debug("Got artist top tracks from Deezer: {0}", artistTracksResponse.data.Count); List deezerTracks = await _deezerApi.GetMultipleTracksAsync(artistTracksResponse.data.Take(desiredArtistTracks).Select(t => t.id)); extendedTracks = deezerTracks.ItemIndex().Select(t => new ExtendedTrack() { DeezerId = long.Parse(t.Item.id), ArtistName = t.Item.artist.name, Title = t.Item.title, Territories = t.Item.available_countries, ISRC = t.Item.isrc, Duration = int.Parse(t.Item.duration), Order = t.Index, Tempo = t.Item.bpm, }).ToList(); } } } if (extendedTracks.Any()) { _logger.Debug("Background lookup for artist {0} got {1} results, caching that.", artistName, extendedTracks.Count); await _cacheProvider.CacheExtendedTopTracksForArtistAsync(artistName, extendedTracks, serviceType); } else { _logger.Debug("Background lookup for artist {0} got no results, caching that.", artistName); _distributedCacheHandler.Put("NoTracks:" + artistName, true); } return extendedTracks; } private List BuildSpotifyTracks(IEnumerable spotifyRawTracks) { return spotifyRawTracks.Where(t => t != null).Select( t => new Track() { ID = -1, Name = t.name, ArtistName = t.artists.First().name, Territories = t.available_markets, SpotifyLink = new SpotifyLink(t.uri), Duration = t.duration_ms/1000, ISRC = GetISRC(t), }).ToList(); } private string GetISRC(SpotifyWebAPI.Model.Track track) { if (track.external_ids != null && track.external_ids.isrc != null) return track.external_ids.isrc; return null; } public async Task> GetExtendedTracksAsync(List spotifyTracks) { var extendedTracks = GetCachedExtendedTracks(spotifyTracks); var nonCachedTracks = spotifyTracks.Where(t => !extendedTracks.Any(ti => ti.SpotifyLinks.Contains(t.SpotifyLink))).ToList(); if (nonCachedTracks.Any()) { var trackIds = spotifyTracks.Select(t => t.SpotifyLink.ExtractTrackID()).Distinct().ToList(); extendedTracks = await FillSpotifyExtendedValuesAsync(trackIds); } return extendedTracks; } private static async Task> FillSpotifyExtendedValuesAsync(List trackIds) { List extendedTracks = new List(); var spotifyTrackInfos = await _spotifyWebApi.GetAllTracksAsync(trackIds); var spotifyTrackInfoDic = spotifyTrackInfos.Where(t => t?.id != null).DistinctBy(k => k.id).ToDictionary(k => k.id, v => v); var trackAudioFeatures = await _spotifyWebApi.GetAllAudioFeatureForTracksAsync(trackIds); Dictionary trackAudioFeaturesDic = new Dictionary(); if (trackAudioFeatures != null) trackAudioFeaturesDic = trackAudioFeatures.Where(t => t?.id != null).DistinctBy(k => k.id).ToDictionary(k => k.id, v => v); var artistId = spotifyTrackInfos.Select(t => t.artists.First().id).Distinct().ToList(); var artists = await _spotifyWebApi.GetAllArtistsAsync(artistId); var artistsDic = artists.ToDictionary(k => k.id, v => v); foreach (var trackId in trackIds) { var spotifyTrackInfo = spotifyTrackInfoDic.GetValueOrDefault(trackId); if (spotifyTrackInfo == null) continue; var spotifyLink = new SpotifyLink(spotifyTrackInfo.uri, urlDecode: true); var track = new ExtendedTrack() { SpotifyLink = spotifyLink, SpotifyLinks = new List() { spotifyLink }, ArtistName = spotifyTrackInfo.artists.FirstOrDefault()?.name, Title = spotifyTrackInfo.name, }; track.Hotness = spotifyTrackInfo.popularity / 100.0; track.ISRC = spotifyTrackInfo.external_ids?.isrc; track.ArtistFamiliarity = spotifyTrackInfo.artists?.First()?.popularity / 100.0; track.ArtistHotnesss = spotifyTrackInfo.artists?.First()?.popularity / 100.0; track.Duration = spotifyTrackInfo.duration_ms / 1000; var firstArtistId = spotifyTrackInfo?.artists?.First().id; if (firstArtistId != null) { var artistInfo = artistsDic.GetValueOrDefault(firstArtistId); if (artistInfo != null) { track.ArtistFamiliarity = artistInfo.popularity / 100.0; track.ArtistHotnesss = artistInfo.popularity / 100.0; } } var audioFeature = trackAudioFeaturesDic?.GetValueOrDefault(trackId); if (audioFeature != null) { track.Energy = audioFeature.energy; track.ArtistMoods = null; track.Danceability = audioFeature.danceability; track.DeezerId = null; track.Duration = audioFeature.duration_ms / 1000; track.FetchCodeVersion = 2; track.FetchDate = DateTime.UtcNow; track.Loudness = audioFeature.loudness; track.Mode = audioFeature.mode; track.SongType = new List(); track.Tempo = audioFeature.tempo; //audioFeature.acousticness //audioFeature.instrumentalness //audioFeature.liveness //audioFeature.speechiness //audioFeature.valence } extendedTracks.Add(track); } return extendedTracks; } private List GetCachedExtendedTracks(List spotifyTracks) { List allCachedExtendedTracks = new List(); var cacheKeys = spotifyTracks.Select(t => BuildExtendedCacheKey(t.SpotifyLink)); var cachedTracks = _distributedCacheHandler.Get(cacheKeys); foreach (var track in spotifyTracks) { var cacheKey = BuildExtendedCacheKey(track.SpotifyLink); if (cachedTracks.ContainsKey(cacheKey)) { var extendedTrack = cachedTracks[cacheKey] as ExtendedTrack; if (extendedTrack != null) { allCachedExtendedTracks.Add(extendedTrack); } } } return allCachedExtendedTracks; } private string BuildExtendedCacheKey(SpotifyLink spotifyLink) { return "CacheExtendedTrack-1.1:" + ExtendedTrack.Version + ":" + spotifyLink.Uri; } private bool IsInTerritory(ExtendedTrack track, string territoryCode) { return (territoryCode == null || EmptyTerritories(track.Territories) || track.Territories.Contains(territoryCode, StringComparer.InvariantCultureIgnoreCase) || track.Territories.Contains("worldwide")); } private bool EmptyTerritories(IEnumerable regions) { IEnumerable list = regions as IList ?? regions.ToList(); return !list.Any() || (list.Count() == 1 && string.IsNullOrWhiteSpace(list.First())); } private bool IsInTerritory(SpotifyWebAPI.Model.Track spotifyTrack, string regionCode) { return (regionCode == null || EmptyTerritories(spotifyTrack.available_markets) || spotifyTrack.available_markets.Contains(regionCode, StringComparer.InvariantCultureIgnoreCase) || spotifyTrack.available_markets.Contains("worldwide")); } public async Task>> GetTopTracksForTagAsync(string tag, bool ignoreCache = false) { var topTracks = await _cacheProvider.FetchTopTrackForTagAsync(tag); if (topTracks == null || ignoreCache) { topTracks = _lastFmMusicInfoStore.GetTopTracksForTag(tag); if (topTracks != null) await _cacheProvider.CacheTopTrackForTagAsync(tag, topTracks); } return topTracks; } public async Task>> GetRelatedTagsAsync(Application application, string tag) { //TODO: Caching List tags; try { tags = await _lastFmMusicInfoStore.GetRelatedTagsAsync(tag); } catch (Exception ex) { ErrorLoggingManager.Instance.LogError(ex); return new List>(); } var tagMatches = _genreManager.MatchTags(application, tags); return tagMatches; } public async Task>> GetExtendedTracksForTag(Genre genre, string regionCode, bool ignoreCache = false) { var groupedTracksForTag = _cacheProvider.FetchExtendedTopTracksForTag(genre.LastFMTag); var hasFetchedTracksForTag = GetHasFetchedTracksForTag(genre.LastFMTag); if (groupedTracksForTag == null || !groupedTracksForTag.Any() || !hasFetchedTracksForTag) { //Get top tracks for a tag on LastFM var lastFmTopTracks = await GetTopTracksForTagAsync(genre.LastFMTag, ignoreCache); var localGroupedTracksForTag = await GetGroupedExtendedTrackAsync(lastFmTopTracks); _cacheProvider.CacheExtendedTopTracksForTag(genre.LastFMTag, localGroupedTracksForTag); SetHasFetchedTracksForTag(genre.LastFMTag); List> extendedTracks = (localGroupedTracksForTag.SelectMany( groupedExtendedTrack => groupedExtendedTrack.SpotifyTracks, (groupedExtendedTrack, spotifyTrack) => new {groupedExtendedTrack, spotifyTrack}) .Where(track => IsInTerritory(track.spotifyTrack, regionCode)) .DistinctBy(t => t.groupedExtendedTrack.MBID) .Select(track => new RankedItem(track.groupedExtendedTrack.ExtendedTrack, track.groupedExtendedTrack.Track.Weight))).ToList(); return extendedTracks; } else { var extendedTracksForTag = groupedTracksForTag .Where(t => IsInTerritory(t.ExtendedTrack, regionCode)) .DistinctBy(t => t.MBID) .Select(t => new RankedItem(t.ExtendedTrack, t.Track.Weight)).ToList(); return extendedTracksForTag; } } private bool GetHasFetchedTracksForTag(string tag) { if (_redisDatabase.IsConnected("IsConnected")) { var key = "HasFetchedTracksForTag:" + tag.ToLowerInvariant(); return _redisDatabase.KeyExists(key); } return false; } private void SetHasFetchedTracksForTag(string tag) { if (_redisDatabase.IsConnected("IsConnected")) { var key = "HasFetchedTracksForTag:" + tag.ToLowerInvariant(); _redisDatabase.StringSet(key, "1", TimeSpan.FromDays(2)); } } } }