using System; using System.Collections.Concurrent; using System.Drawing.Imaging; using System.IO; using System.Linq; using System.Net; using System.Net.Http; using System.Threading.Tasks; using CacheCow.Client; using MoreLinq; using NLog; using Sony.Filtr.ApolloAPI; using Sony.Filtr.Contracts.Abstractions; using Sony.Filtr.Contracts.Definitions; using Sony.Filtr.Contracts.Entities; using Sony.Filtr.Core.Factory; using Sony.Filtr.Core.Releases; using Sony.Filtr.Core.Settings; using Sony.Filtr.DistributedCaching; using Sony.Filtr.PlaylistGeneration; using Sony.Filtr.Search; using Sony.Filtr.SpotifyWebAPI.Model; using Sony.Filtr.Utility.Extensions; using Sony.Filtr.VendorToSpotify; using Track = Sony.Filtr.Contracts.Entities.Track; namespace Sony.Filtr.Tasks.Tasks.PublicApi { public class UpdateSpotifyReleases : IScheduledTask { private readonly ReleasesManager _releasesManager; private readonly IApplicationInstanceManager _applicationInstanceManager; private readonly ImageFactory _imageFactory; private readonly PlaylistGenerationManager _playlistGenerationManager; private readonly DistributedCacheHandler _distributedCacheHandler; private readonly SearchManager _searchManager; private readonly DistributedCachingStore _distributedCachingStore; private readonly ICDNCacheManager _cdnCacheManager; private readonly SettingsManager _settingsManager; private readonly Logger _logger; private readonly IApolloWebApi _vendorApi; public UpdateSpotifyReleases(IApolloWebApi vendorApi, ReleasesManager releasesManager, IApplicationInstanceManager applicationInstanceManager, ImageFactory imageFactory, PlaylistGenerationManager playlistGenerationManager, DistributedCacheHandler distributedCacheHandler, SearchManager searchManager, DistributedCachingStore distributedCachingStore, ICDNCacheManager cdnCacheManager, SettingsManager settingsManager) { _vendorApi = vendorApi; _releasesManager = releasesManager; _applicationInstanceManager = applicationInstanceManager; _imageFactory = imageFactory; _playlistGenerationManager = playlistGenerationManager; _distributedCacheHandler = distributedCacheHandler; _searchManager = searchManager; _distributedCachingStore = distributedCachingStore; _cdnCacheManager = cdnCacheManager; _settingsManager = settingsManager; _logger = LogManager.GetLogger("SpotifyReleases"); } public async Task ExecuteAsync(Guid scheduledTaskLogId) { var taskInfo = new ScheduledTaskLog(); var spotifyAlbumReleaseDataTask = new ScheduledTaskDataLog("SpotifyReleases-Album", DateTimeOffset.UtcNow, scheduledTaskLogId); var spotifySingleReleaseDataTask = new ScheduledTaskDataLog("SpotifyReleases-Single", DateTimeOffset.UtcNow, scheduledTaskLogId); _logger.Info("Begin UpdateSpotifyReleases"); ConcurrentBag allReleases = new ConcurrentBag(); ConcurrentBag allRemovedReleases = new ConcurrentBag(); var applications = _applicationInstanceManager.GetApplications(); ServicePointManager.DefaultConnectionLimit = 100; await applications.ForEachAsync(5, async app => { _logger.Info("Getting settings for " + app.Name); var albumUri = _settingsManager.GetStringSetting("ReleaseAlbumPlaylistUrl", app); var singleUri = _settingsManager.GetStringSetting("ReleasePlaylistUrl", app); ConcurrentBag releases = new ConcurrentBag(); try { var existingReleases = _releasesManager.GetReleases(app).ToList(); var albumReleases = await GetReleasesAsync(app, albumUri, ReleaseType.Album); _logger.Info("Found {0} album releases for {1}", albumReleases.Count, app.Name); albumReleases.ForEach(r=> releases.Add(r)); var albumReleaseLinks = albumReleases.ToDictionary(k => k.TrackLink, v => v.ID); var removedAlbumReleases = existingReleases.Where(a => a.ReleaseType == ReleaseType.Album && !albumReleaseLinks.ContainsKey(a.TrackLink)); var singleReleases = await GetReleasesAsync(app, singleUri, ReleaseType.Single); _logger.Info("Found {0} single releases for {1}", singleReleases.Count, app.Name); singleReleases.ForEach(r => releases.Add(r)); var singleReleaseLinks = singleReleases.ToDictionary(k => k.TrackLink, v => v.ID); var removedSingleReleases = existingReleases.Where(a => a.ReleaseType == ReleaseType.Single && !singleReleaseLinks.ContainsKey(a.TrackLink)); var removedReleases = removedAlbumReleases.Union(removedSingleReleases).ToList(); removedReleases.ForEach(allRemovedReleases.Add); releases.ForEach(allReleases.Add); _logger.Info("Setting releases for {0}", app.Name); _releasesManager.SetReleases(releases.Where(r=> r != null).ToList(), app); } catch (Exception ex) { spotifyAlbumReleaseDataTask.Error = spotifySingleReleaseDataTask.Error = true; spotifyAlbumReleaseDataTask.ErrorMessage = spotifySingleReleaseDataTask.ErrorMessage = ex.Message; _logger.Error("Could not get and set releases for market {0}. Exception: {1}", app.Name, ex.ToString()); } }); _logger.Info("Begin indexing"); if (allRemovedReleases.Any()) { _logger.Debug("Removing releases from index: " + string.Join(",", allRemovedReleases.Select(r=> r.ID))); await _searchManager.DeleteReleasesFromIndexAsync(allRemovedReleases.ToList()); } if (allReleases.Any()) { _logger.Debug("Indexing releases: " + string.Join(",", allReleases.Select(r => r.ID))); await _searchManager.IndexReleasesAsync(allReleases.ToList()); } await _cdnCacheManager.PurgeReleaseEndpointsAsync(); _logger.Info("Done with UpdateNewReleases"); spotifyAlbumReleaseDataTask.Rows = allReleases.Where(a => a.ReleaseType == ReleaseType.Album).DistinctBy(r => r.SpotifyLinkUri).Count(); spotifySingleReleaseDataTask.Rows = allReleases.Where(a => a.ReleaseType == ReleaseType.Single).DistinctBy(r => r.SpotifyLinkUri).Count(); spotifyAlbumReleaseDataTask.Finished = spotifySingleReleaseDataTask.Finished = DateTimeOffset.UtcNow; taskInfo.DataLogs.Add(spotifyAlbumReleaseDataTask); taskInfo.DataLogs.Add(spotifySingleReleaseDataTask); return taskInfo; } private async Task> GetReleasesAsync(Application app, string spotifyUri, ReleaseType releaseType) { ConcurrentBag releases = new ConcurrentBag(); if (string.IsNullOrWhiteSpace(spotifyUri)) return releases; var link = new SpotifyLink(spotifyUri); const string fields = "items.track(name,album.uri,album.name,album.available_markets,artists.name,artists.uri,album.id,album.images(height,width,url),uri),total"; //var playlistTracks = (await _spotifyWebApi.GetAllPlaylistTracksByIdAsync(link.ExtractPlaylistID(), fields)).ToList(); var playlistTracks = (await _vendorApi.GetAllPlaylistTracksByIdAsync(link.ExtractPlaylistID(), fields)).ToList(); if (playlistTracks == null) { _logger.Warn("Could not get tracks for playlist {0}", spotifyUri); return releases; } playlistTracks = playlistTracks.Where(t => t != null && t.track != null).ToList(); var tracks = playlistTracks.DistinctBy(t => t.track.album.uri).ToList(); if (playlistTracks.Any()) { ConcurrentBag allExtendedTracks = new ConcurrentBag(); try { foreach (var tracksBatch in tracks.Batch(50)) { var extendedTracks = await _playlistGenerationManager.GetExtendedTracksAsync(tracksBatch.Select(t => new Track() { SpotifyLink = new SpotifyLink(t.track.uri) }).ToList()); if (extendedTracks != null && extendedTracks.Any()) { foreach (var extendedTrack in extendedTracks) { allExtendedTracks.Add(extendedTrack); } } } } catch (Exception ex) { _logger.Error(ex, "Could not get ExtendedTrack, exception: {0}"); } await tracks.ItemIndex().ForEachAsync(5, async trackIndex => { var track = trackIndex.Item; try { var album = track.track.album; var albumAvailability = album.available_markets; var artistName = track.track.artists.First().name; var artistUri = track.track.artists.First().uri; var imageFilename = await TrySaveImageAsync(album.ToSpotify()); var release = new Release() { AlbumLink = new SpotifyLink(album.uri), TrackLink = new SpotifyLink(track.track.uri), ArtistName = artistName, ArtistUri = artistUri, AlbumName = album.name, TrackName = track.track.name, Availability = albumAvailability, ReleaseType = releaseType, ApplicationID = app.ID, ImageFilename = imageFilename, Index = trackIndex.Index, }; var extendedTrack = allExtendedTracks.FirstOrDefault(t => t.SpotifyLinks.Contains(new SpotifyLink(track.track.uri))); if (extendedTrack != null) { release.Energy = extendedTrack.Energy; release.Hotness = extendedTrack.Hotness; release.Danceability = extendedTrack.Danceability; release.Tempo = extendedTrack.Tempo; } releases.Add(release); } catch (Exception ex) { _logger.Error("Failed to update release for {0}. Exception: {1}", track.track.name, ex); } }); } return releases; } private async Task TrySaveImageAsync(Album album) { var albumImage = album.images.OrderByDescending(i => i.width).FirstOrDefault(); if (albumImage == null) return null; string imageUrl = albumImage.url; string filename = null; try { var cacheKey = "SpotifyReleaseAlbumImage:" + album.id; filename = _distributedCacheHandler.Get(cacheKey) as string; if (filename == null) { filename = "album-" + album.id + ".jpeg"; _logger.Info("Getting image for album {0}", album.id); var httpClient = GetDefaultHttpClient(); var response = await httpClient.GetAsync(imageUrl); var responseStream = await response.Content.ReadAsStreamAsync(); using (var image = System.Drawing.Image.FromStream(responseStream)) using (var output = new MemoryStream()) { image.Save(output, ImageFormat.Jpeg); _logger.Info("Saving image to Amazon for album {0}", album.id); await _imageFactory.SaveImageAsync(filename, output); } _distributedCacheHandler.Put(cacheKey, filename, DateTime.Now.AddDays(5)); } else { //_logger.Debug("Skipping album images since its cached"); } return filename; } catch (Exception ex) { _logger.Error("Could not get image for {0}. Exception: {1}", album.name, ex.ToString()); } return filename; } private HttpClient GetDefaultHttpClient() { return new HttpClient(new CachingHandler(_distributedCachingStore) { InnerHandler = new HttpClientHandler()}); } } }