using Newtonsoft.Json; using Newtonsoft.Json.Linq; using NLog; using Sony.Filtr.ApolloAPI.Models; using Sony.Filtr.Utility.Extensions; using System; using System.Collections.Generic; using System.Linq; using System.Net.Http; using System.Threading.Tasks; using System.Web; namespace Sony.Filtr.ApolloAPI { public class ApolloWebApi : ApolloWebApiBase, IApolloWebApi { private readonly ApolloApiSettings Settings; public const int SpotifyMaxTracksPerRequest = 100; public const int MaxArtistBatchSize = 50; public const int MaxPlaylistsForUserBatchSize = 50; public ApolloWebApi(HttpClient httpClient, ApolloApiSettings settings) : base(httpClient) { this.Settings = settings; } protected override string GetLoggerName() { return "ApolloWebApi"; } private async Task GetByBatches( int batchSize, Func getTotalFunc, Func> spotifyFunc, Func> getItemsFunc, TimeSpan cacheExpiration) { var initialResponse = await spotifyFunc(new Batch(0, batchSize), cacheExpiration); if (initialResponse == null) { return Array.Empty(); } var result = new List(getItemsFunc(initialResponse)); if (getTotalFunc(initialResponse) > result.Count) { var responses = await this.GetBatchResponses( getTotalFunc(initialResponse).Batch(batchSize).Skip(1), async (batch, expiration) => getItemsFunc(await spotifyFunc(batch, expiration)), cacheExpiration); result.AddRange(responses); } return result.ToArray(); } private async Task GetBatchResponses( IEnumerable batches, Func>> spotifyFunc, TimeSpan cacheExpiration) { var tasks = batches.Select(async b => { var response = await spotifyFunc(b, cacheExpiration); return BatchResponse.CreateSuccess(response, b); }).ToArray(); await tasks.EnsureAllSuccessAsync(); return tasks.Select(t => t.Result).OrderBy(r => r.Batch.Offset).SelectMany(r => r.Items).ToArray(); } public async Task GetAllBrowseCategoriesAsync(string region = null) { return await this.GetAllBrowseCategoriesAsync(NoCacheTime, region); } public async Task GetAllBrowseCategoriesAsync(TimeSpan cacheExpiration, string region = null) { return await this.GetByBatches( MaxPlaylistsForUserBatchSize, r => r.categories.total, async (batch, expiration) => await GetBrowseCategoriesAsync(expiration, region, limit: batch.Limit, offset: batch.Offset), r => r.categories.items, cacheExpiration); } public async Task GetPlaylistFavorites() { string url = "user-data-api/v1/joint/favorites/?application=apollo&type=playlist"; FavoritePlaylistsResponse response = await this.GetFromApollo(url, ParseResponse, NoCacheTime); return response.Normalize(); } public async Task GetBrowseCategoriesAsync(TimeSpan cacheExpiration, string country = null, int? limit = null, int? offset = null) { string pathAndQuery = "vendor-api/spotify/v1/browse/categories"; var parameters = HttpUtility.ParseQueryString(String.Empty); if (!string.IsNullOrWhiteSpace(country)) { parameters["country"] = country; } if (limit.HasValue) { parameters["limit"] = limit.ToString(); } if (offset.HasValue) { parameters["offset"] = offset.ToString(); } pathAndQuery += $"?{parameters.ToString()}"; return await this.GetFromApollo(pathAndQuery, ParseResponse< SpotifyBrowseCategoriesResponse>, cacheExpiration); } public async Task GetAllBrowseCategoryPlaylistsAsync(string categoryId, string region = null) { return await this.GetAllBrowseCategoryPlaylistsAsync(categoryId, NoCacheTime, region); } public async Task GetAllBrowseCategoryPlaylistsAsync(string categoryId, TimeSpan cacheExpiration, string region = null) { return await this.GetByBatches( MaxPlaylistsForUserBatchSize, r => r.playlists.total, async (batch, expiration) => await GetBrowseCategoryPlaylistsAsync(categoryId, expiration, region, limit: batch.Limit, offset: batch.Offset), r => r?.playlists?.items, cacheExpiration); } public async Task GetBrowseCategoryPlaylistsAsync(string categoryId, TimeSpan cacheExpiration, string country = null, int? limit = null, int? offset = null) { string pathAndQuery = $"vendor-api/spotify/v1/browse/categories/{categoryId}/playlists"; var parameters = HttpUtility.ParseQueryString(String.Empty); if (!string.IsNullOrWhiteSpace(country)) { parameters["country"] = country; } if (limit.HasValue) { parameters["limit"] = limit.ToString(); } if (offset.HasValue) { parameters["offset"] = offset.ToString(); } pathAndQuery += $"?{parameters.ToString()}"; return await this.GetFromApollo(pathAndQuery, ParseResponse< SpotifyBrowsePlaylistsResponse>, cacheExpiration); } public async Task GetBrowseFeaturedPlaylistsAsync(string country = null) { return await this.GetBrowseFeaturedPlaylistsAsync(NoCacheTime, country); } public async Task GetBrowseFeaturedPlaylistsAsync(TimeSpan cacheExpiration, string country = null) { string pathAndQuery = "vendor-api/spotify/v1/browse/featured-playlists"; var parameters = HttpUtility.ParseQueryString(String.Empty); if (!string.IsNullOrWhiteSpace(country)) { parameters["country"] = country; } parameters["limit"] = 50.ToString(); // 50 is max as specified in the API documentation 2015-02-17 pathAndQuery += $"?{parameters.ToString()}"; return await this.GetFromApollo(pathAndQuery, ParseResponse< SpotifyBrowsePlaylistsResponse>, cacheExpiration); } public async Task GetUserAsync(string spotifyUsername, string fields = null) { return await this.GetUserAsync(spotifyUsername, NoCacheTime, fields); } public async Task GetUserAsync(string spotifyUsername, TimeSpan cacheExpiration, string fields = null) { string pathAndQuery = "vendor-api/spotify/v1/users/"; var parameters = HttpUtility.ParseQueryString(String.Empty); if (!String.IsNullOrWhiteSpace(fields)) { parameters["fields"] = fields; } pathAndQuery += $"?{parameters.ToString()}"; return await this.GetFromApollo(pathAndQuery, ParseResponse< SpotifyUserResponse>, cacheExpiration); } public async Task GetAllPublicPlaylistsAsync(string userName, string fields = null) { return await this.GetAllPublicPlaylistsAsync(userName, NoCacheTime, fields); } public async Task GetAllPublicPlaylistsAsync(string userName, TimeSpan cacheExpiration, string fields = null) { return await this.GetByBatches( MaxPlaylistsForUserBatchSize, r => r.total, async (batch, expiration) => await GetPublicPlaylistsAsync(userName, expiration, fields, limit: batch.Limit, offset: batch.Offset), r => r.items, cacheExpiration); } public async Task GetPublicPlaylistsAsync(string username, TimeSpan cacheExpiration, string fields = null, int? limit = null, int? offset = null) { string pathAndQuery = $"vendor-api/spotify/v1/users/{username}/playlists"; var parameters = HttpUtility.ParseQueryString(String.Empty); if (limit != null) { parameters["limit"] = limit.Value.ToString(); } if (offset != null) { parameters["offset"] = offset.Value.ToString(); } if (!String.IsNullOrWhiteSpace(fields)) { parameters["fields"] = fields; } pathAndQuery += $"?{parameters.ToString()}"; return await this.GetFromApollo(pathAndQuery, ParseResponse< SpotifyPublicPlaylistsResponse>, cacheExpiration); } public async Task> GetAllArtistsAsync(IEnumerable spotifyArtistIds) { return await this.GetAllArtistsAsync(spotifyArtistIds, NoCacheTime); } public async Task> GetAllArtistsAsync(IEnumerable spotifyArtistIds, TimeSpan cacheExpiration) { var responses = await this.GetBatches( spotifyArtistIds.SplitOnBatches(MaxArtistBatchSize), async (batch, expiration) => await this.GetArtistsByIdsAsync(batch, expiration), cacheExpiration); return responses .Where(r => r.artists != null && r.artists.Any() && r.artists.First() != null) .SelectMany(r => r.artists) .ToList(); } private async Task GetArtistsByIdsAsync(IEnumerable ids, TimeSpan cacheExpiration) { string pathAndQuery = $"vendor-api/spotify/v1/artists?"; pathAndQuery += $"ids={String.Join(",", ids)}"; return await this.GetFromApollo(pathAndQuery, ParseResponse< ArtistResponse>, cacheExpiration); } public async Task SearchArtistsAsync(string artistName, string fields = null, int? limit = null, int? offset = null) { return await this.SearchArtistsAsync(artistName, NoCacheTime, fields, limit, offset); } public async Task SearchArtistsAsync(string artistName, TimeSpan cacheExpiration, string fields = null, int? limit = null, int? offset = null) { var pathAndQuery = "vendor-api/spotify/v1/search"; var parameters = HttpUtility.ParseQueryString(String.Empty); parameters["type"] = "artist"; parameters["q"] = artistName; if (limit != null) { parameters["limit"] = limit.Value.ToString(); } if (offset != null) { parameters["offset"] = offset.Value.ToString(); } if (fields != null) { parameters["fields"] = fields; } pathAndQuery += $"?{parameters.ToString()}"; return await this.GetFromApollo(pathAndQuery, ParseResponse, cacheExpiration); } public async Task IsSonyTrack(IEnumerable trackIds, string market) { return await this.IsSonyTrack(trackIds, market, NoCacheTime); } public async Task IsSonyTrack(IEnumerable trackIds, string market, TimeSpan cacheExpiration) { var responses = await this.GetBatches( trackIds.SplitOnBatches(this.Settings.MaxSpotifyIdsInIsSonyRequest), async (batch, expiration) => await this.IsSonyTrackBatchAsync(batch, market, expiration), cacheExpiration); return responses.SelectMany(r => r).ToArray(); } public async Task GetSpotifyTracks(IEnumerable trackIds, string market) { return await this.GetSpotifyTracks(trackIds, market, NoCacheTime); } public async Task GetSpotifyTracks(IEnumerable trackIds, string market, TimeSpan cacheExpiration) { var responses = await this.GetBatches( trackIds.SplitOnBatches(this.Settings.MaxSpotifyIdsInGetTracksRequest), async (batch, expiration) => await this.GetTracksBatchAsync(batch, market, expiration), cacheExpiration); var response = new GetTracksResponse() { tracks = responses.SelectMany(r => r.tracks).ToArray() }; foreach (var r in responses) { foreach (var requestToHeader in r.RequestHeaders) { response.AddHeaders(requestToHeader.Key, requestToHeader.Value); } } return response; } public async Task GetSpotifyPlaylistByIdAsync(string playlistId, string fields = null) { return await this.GetSpotifyPlaylistByIdAsync(playlistId, NoCacheTime, fields); } public async Task GetSpotifyPlaylistByIdAsync(string playlistId, TimeSpan cacheExpiration, string fields = null) { string pathAndQuery = $"vendor-api/spotify/v1/playlists/{playlistId}"; if (fields != null) { var parameters = HttpUtility.ParseQueryString(String.Empty); parameters["fields"] = fields; pathAndQuery += $"?{parameters.ToString()}"; } return await this.GetFromApollo( pathAndQuery, ParseResponse< SpotifyPlaylist>, cacheExpiration); } private static bool DoNotSkipTracksLoad(SpotifyPlaylist r) { return false; } public async Task GetPlaylistByIdWithAllTracksAsync(string playlistId, string playlistFields = null, string market = null) { return await this.GetPlaylistByIdWithAllTracksAsync(playlistId, NoCacheTime, playlistFields, market); } public async Task GetPlaylistByIdWithAllTracksAsync(string playlistId, TimeSpan cacheExpiration, string playlistFields = null, string market = null) { return await this.GetPlaylistByIdWithAllTracksWithFilterAsync(playlistId, playlistFields, market, cacheExpiration, DoNotSkipTracksLoad); } public async Task GetPlaylistByIdWithAllTracksHavingSnapshotIdAsync(string playlistId, string snapshotId, string playlistFields = null, string market = null) { return await this.GetPlaylistByIdWithAllTracksHavingSnapshotIdAsync(playlistId, snapshotId, NoCacheTime, playlistFields, market); } public async Task GetPlaylistByIdWithAllTracksHavingSnapshotIdAsync(string playlistId, string snapshotId, TimeSpan cacheExpiration, string playlistFields = null, string market = null) { var response = await this.GetPlaylistByIdWithAllTracksWithFilterAsync(playlistId, playlistFields, market, cacheExpiration, p => !String.IsNullOrWhiteSpace(p.snapshot_id) && p.snapshot_id.Equals(snapshotId)); if (response != null) { response.HasSnapshotIdChanged = !String.Equals(response.snapshot_id, snapshotId); } return response; } private async Task GetPlaylistByIdWithAllTracksWithFilterAsync(string playlistId, string playlistFields, string market, TimeSpan cacheExpiration, Func shouldSkipLoadingTracksForPlaylist) { SpotifyPlaylist playlist = await this.GetSpotifyPlaylistByIdAsync(playlistId, cacheExpiration, playlistFields); if (playlist == null) { return null; } if (shouldSkipLoadingTracksForPlaylist(playlist)) { return playlist; } //Spotify returns first SpotifyMaxTracksPerRequest tracks, if present if (playlist.Tracks.total <= SpotifyMaxTracksPerRequest) { return playlist; } var responses = await this.GetBatchResponses( playlist.Tracks.total.Batch(SpotifyMaxTracksPerRequest).Skip(1).ToArray(), async (batch, expiration) => new SpotifyPlaylistTracks[] { await GetPlaylistTracksByIdAsync(playlistId, expiration, limit: batch.Limit, offset: batch.Offset, market: market) }, cacheExpiration); foreach (var response in responses) { foreach (var requestToHeader in response.RequestHeaders) { playlist.AddHeaders(requestToHeader.Key, requestToHeader.Value); } playlist.AddRawResponseToModel(response.RawResponse.Key, response.RawResponse.Value); } playlist.Tracks.items.AddRange(responses.SelectMany(r => r.items)); playlist.Tracks.offset = 0; playlist.Tracks.limit = playlist.Tracks.items.Count(); return playlist; } public async Task GetAllPlaylistTracksByIdAsync(string playlistId, string fields = null, string market = null) { return await this.GetAllPlaylistTracksByIdAsync(playlistId, NoCacheTime, fields, market); } public async Task GetAllPlaylistTracksByIdAsync(string playlistId, TimeSpan cacheExpiration, string fields = null, string market = null) { var playlist = await this.GetPlaylistByIdWithAllTracksAsync(playlistId, cacheExpiration, fields, market); return playlist == null ? new SpotifyPlaylistTrackItem[0] : playlist.Tracks.items.ToArray(); } private async Task GetPlaylistTracksByIdAsync(string playlistId, TimeSpan cacheExpiration, string fields = null, string market = null, Nullable offset = null, Nullable limit = null) { var path = $"vendor-api/spotify/v1/playlists/{playlistId}/tracks"; var parameters = HttpUtility.ParseQueryString(string.Empty); if (fields != null) { parameters["fields"] = fields; } if (limit.HasValue) { parameters["limit"] = limit.Value.ToString(); } if (offset.HasValue) { parameters["offset"] = offset.Value.ToString(); } if (market != null) { parameters["market"] = market; } string finalUrl = $"{path}?{parameters.ToString()}"; return await GetFromApollo(finalUrl, ParseResponse< SpotifyPlaylistTracks>, cacheExpiration); } public async Task GetAllAlbumInfoAsync(IEnumerable ids, string market = null, string fields = null) { return await this.GetAllAlbumInfoAsync(ids, NoCacheTime, market, fields); } public async Task GetAllAlbumInfoAsync(IEnumerable ids, TimeSpan cacheExpiration, string market = null, string fields = null) { var responses = await this.GetBatches( ids.SplitOnBatches(this.Settings.MaxAlbumIdsInRequest), async (batch, expiration) => await this.GetAlbumInfoBatchAsync(batch, expiration, market), cacheExpiration); return responses.SelectMany(r => r).ToArray(); } public async Task GetAllAudioFeatureForTracksAsync(IEnumerable trackIds) { return await this.GetAllAudioFeatureForTracksAsync(trackIds, NoCacheTime); } public async Task GetAllAudioFeatureForTracksAsync(IEnumerable trackIds, TimeSpan cacheExpiration) { var responses = await this.GetBatches( trackIds.SplitOnBatches(100), async (batch, expiration) => await this.GetAudioFeatureForTracksAsync(batch, expiration), cacheExpiration); return responses.SelectMany(r => r).ToArray(); } private async Task> GetAudioFeatureForTracksAsync(IEnumerable ids, TimeSpan cacheExpiration) { string url = $"/vendor-api/spotify/v1/audio-features?{string.Join("&", ids.Select(id => $"ids={id}"))}"; return await this.GetFromApollo( url, this.ParseAudioFeatures, cacheExpiration); } private async Task> GetAlbumInfoBatchAsync(IEnumerable ids, TimeSpan cacheExpiration, string market = null) { string url = $"/vendor-api/spotify/v1/albums?{string.Join("&", ids.Select(id => $"ids={id}"))}"; if (!String.IsNullOrWhiteSpace(market)) { url += $"&market={market}"; } return await this.GetFromApollo( url, this.ParseSpotifyAlbums, cacheExpiration); } private async Task GetTracksBatchAsync(IEnumerable trackIds, string market, TimeSpan cacheExpiration) { string queryString = $"ids={string.Join(",", trackIds)}"; if (!String.IsNullOrWhiteSpace(market)) { queryString += $"&market={market}"; } return await this.GetFromApollo( $"/vendor-api/spotify/v1/tracks?{queryString}", ParseResponse< GetTracksResponse>, cacheExpiration); } private async Task> IsSonyTrackBatchAsync(IEnumerable trackIds, string market, TimeSpan cacheExpiration) { return await this.GetFromApollo( $"/apollo-api/is-sony/?{string.Join("&", trackIds.Select(id => $"spotify_ids={id}"))}&market={TreatNullAsGlobal(market)}", this.ParseIsSonyTrackJson, cacheExpiration); } private IEnumerable ParseIsSonyTrackJson(string responseString) { JObject jObject = null; try { jObject = JObject.Parse(responseString); } catch (JsonReaderException ex) { throw new ApolloWebAPIException($"Could not parse 'is-sony' response. Response: '{responseString}'", ex); } return jObject .Children() .OfType() .Select(p => new IsSonyTrack(p.Name, Boolean.Parse(p.Value.ToString()))) .ToArray(); } private IEnumerable ParseSpotifyAlbums(string responseString) { try { var parsed = JsonConvert.DeserializeObject(responseString); return parsed == null ? Enumerable.Empty() : parsed.albums; } catch (Exception ex) { throw new ApolloWebAPIException($"Could not parse 'get albums by ids' response. Response: '{responseString}'"); } } private IEnumerable ParseAudioFeatures(string responseString) { try { var parsed = JsonConvert.DeserializeObject(responseString); return parsed == null ? Enumerable.Empty() : parsed.audio_features; } catch (Exception ex) { throw new ApolloWebAPIException($"Could not parse 'get audio features by ids' response. Response: '{responseString}'"); } } private static string TreatNullAsGlobal(string market) { return market == null ? "global" : market; } private async Task GetBatches(IEnumerable input, Func> spotifyFunc, TimeSpan cacheExpiration) { var tasks = input.Select(async i => await spotifyFunc(i, cacheExpiration)).ToList(); await tasks.EnsureAllSuccessAsync(); return tasks.Select(t => t.Result).ToArray(); } } public class GetTracksResponse : HeaderTracker { public SpotifyTrack[] tracks { get; set; } } class GetAlbumsResponse { public SpotifyAlbum[] albums { get; set; } } class AudioFeatureResponse { public List audio_features { get; set; } } public class BatchResponse { public readonly TResponse[] Items; public readonly Batch Batch; public readonly Exception Exception; public bool Success { get { return this.Exception == null; } } private BatchResponse(IEnumerable items, Batch batch, Exception ex) { this.Items = items.ToArray(); this.Exception = ex; this.Batch = batch; } public static BatchResponse CreateSuccess(IEnumerable items, Batch batch) { return new BatchResponse(items, batch, null); } public static BatchResponse CreateError(Exception ex, Batch batch) { return new BatchResponse(new TResponse[0], batch, ex); } } }