using System; using System.Collections.Concurrent; using System.Collections.Generic; using System.Linq; using System.Net; using System.Net.Http; using System.Text; using System.Threading.Tasks; using Flurl; using MoreLinq; using Newtonsoft.Json; using NLog; using Sony.Filtr.Contracts.Abstractions; using Sony.Filtr.Contracts.Entities; using Sony.Filtr.SpotifyWebAPI.Model; using Sony.Filtr.Utility; using Sony.Filtr.Utility.Extensions; using Sony.Filtr.Utility.HttpHandlers; using Artist = Sony.Filtr.SpotifyWebAPI.Model.Artist; using Track = Sony.Filtr.SpotifyWebAPI.Model.Track; namespace Sony.Filtr.SpotifyWebAPI { public class SpotifyWebApi { private readonly string _clientId; private readonly string _clientSecret; private readonly HttpClient _client; protected const string BaseUrl = "https://api.spotify.com/"; public const int MaxArtistBatchSize = 50; public const int SpotifyMaxTracksPerRequest = 100; private readonly Logger _logger; public SpotifyWebApi(string clientId, string clientSecret) : this(BuildClientCredentialsHttpClient(clientId, clientSecret)) { _clientId = clientId; _clientSecret = clientSecret; this._logger = LogManager.GetLogger("SpotifyWebApi"); this._logger.Info($"UpdatePlaylistInfo_SpotifyWebApi_ThreadRateLimit: {GetThreadRateLimit()}"); } protected SpotifyWebApi(HttpClient httpClient) { _client = httpClient; } private static int GetThreadRateLimit() { return Maybe.GetAppSettingsIntOrDefault("UpdatePlaylistInfo_SpotifyWebApi_ThreadRateLimit", 5); } private static int GetRetryTimeout() { return Maybe.GetAppSettingsIntOrDefault("SpotifyWebApi_RetryHttpHandler_Timeout_Seconds", 10); } private static HttpClient BuildClientCredentialsHttpClient(string clientId, string clientSecret) { var baseHandler = new HttpClientHandler() { UseCookies = false, AutomaticDecompression = DecompressionMethods.Deflate | DecompressionMethods.GZip }; var rateLimitHandler = new ThreadRateLimitHttpHandler(GetThreadRateLimit()) { InnerHandler = baseHandler }; var retryHandler = new RetryHttpHandler(2, TimeSpan.FromSeconds(GetRetryTimeout()), TimeSpan.FromSeconds(1), rateLimitHandler); var authHandler = new SpotifyAuthClientCredentialsHttpMessageHandler(clientId, clientSecret, retryHandler); return new HttpClient(authHandler) { BaseAddress = new Uri(BaseUrl) }; } protected HttpClient GetDefaultClient() { return _client; } public async Task GetAllTracksAsync(IEnumerable spotifyTrackIds, string market = null) { var responses = await this.GetBatches(spotifyTrackIds.Batch(50), async b => await this.GetTracksFromSpotifyAsync(b, market)); return responses.SelectMany(r => r.tracks).ToArray(); } private async Task GetTracksFromSpotifyAsync(IEnumerable spotifyTrackIds, string market) { var url = new Url(string.Format("/v1/tracks/?ids={0}", string.Join(",", spotifyTrackIds))); if (market != null) { url.SetQueryParam("market", market); } return await this.GetDataFromSpotify(url); } public async Task> GetAllArtistsAsync(IEnumerable spotifyArtistIds) { var responses = await this.GetBatches(spotifyArtistIds.Distinct().Batch(MaxArtistBatchSize), this.GetArtistsByIdsAsync); return responses .Where(r => r.artists != null && r.artists.Any() && r.artists.First() != null) .SelectMany(r => r.artists) .ToList(); } private async Task GetArtistsByIdsAsync(IEnumerable ids) { var url = new Url($"/v1/artists/"); url = url.SetQueryParam("ids", string.Join(",", ids)); return await this.GetDataFromSpotify(url); } public async Task GetPlaylistByIdAsync(string playlistId, string fields = null, string market = null) { var url = new Url("v1/playlists").AppendPathSegment(playlistId); if (fields != null) { url.SetQueryParam("fields", fields); } if (market != null) { url = url.SetQueryParam("market", market); } return await this.GetDataFromSpotify(url); } public async Task GetPlaylistTracksByIdAsync(string playlistId, string fields = null, int? limit = null, int? offset = null, string market = null) { var url = new Url("v1").AppendPathSegments("playlists", playlistId, "tracks"); if (fields != null) url = url.SetQueryParam("fields", fields); if (limit != null) url = url.SetQueryParam("limit", limit); if (offset != null) url = url.SetQueryParam("offset", offset); if (market != null) url = url.SetQueryParam("market", market); return await this.GetDataFromSpotify(url); } public async Task GetAllPlaylistTracksByIdAsync(string playlistId, string fields = null, string market = null) { return await this.GetByBatches( 100, r => r.total, async batch => await GetPlaylistTracksByIdAsync(playlistId, fields, limit: batch.Limit, offset: batch.Offset, market), r => r.items); } private static bool DoNotSkipTracksLoad(PlaylistResponse r) { return false; } public async Task GetPlaylistByIdWithAllTracksAsync(string playlistId, string playlistFields = null, string market = null) { return await this.GetPlaylistByIdWithAllTracksWithFilterAsync(playlistId, playlistFields, market, DoNotSkipTracksLoad); } private async Task GetPlaylistByIdWithAllTracksWithFilterAsync(string playlistId, string playlistFields, string market, Func shouldSkipLoadingTracksForPlaylist) { PlaylistResponse playlist = await this.GetPlaylistByIdAsync(playlistId, playlistFields); if (playlist == null) { return null; } if (shouldSkipLoadingTracksForPlaylist(playlist)) { return playlist; } //Spotify returns first SpotifyMaxTracksPerRequest tracks, if present if (playlist.Tracks.total <= SpotifyMaxTracksPerRequest) { return playlist; } var responses = await this.GetBatchResponses( playlist.Tracks.total.Batch(SpotifyMaxTracksPerRequest).Skip(1).ToArray(), async batch => (await GetPlaylistTracksByIdAsync(playlistId, limit: batch.Limit, offset: batch.Offset, market: market)).items); playlist.Tracks.items.AddRange(responses); playlist.Tracks.offset = 0; playlist.Tracks.limit = playlist.Tracks.items.Count(); return playlist; } public async Task GetPlaylistByIdWithAllTracksHavingSnapshotIdAsync(string playlistId, string snapshotId, string playlistFields = null, string market = null) { var playlist = await this.GetPlaylistByIdWithAllTracksWithFilterAsync( playlistId, playlistFields, market, r => !String.IsNullOrWhiteSpace(r.snapshot_id) && r.snapshot_id.Equals(snapshotId)); if (playlist != null) { playlist.HasSnapshotIdChanged = !String.Equals(playlist.snapshot_id, snapshotId); } return playlist; } public async Task GetAllPublicPlaylistsAsync(string userName, string fields = null) { return await this.GetByBatches( 50, r => r.total, async batch => await GetPublicPlaylistsAsync(userName, fields, limit: batch.Limit, offset: batch.Offset), r => r.items); } public async Task> GetAllAlbumInfoAsync(IEnumerable ids, string market = null, string fields = null) { var responses = await this.GetBatches(ids.Batch(20), async b => await this.GetAlbumInfoAsync(b, market, fields)); return responses.SelectMany(r => r.albums).Where(a => a != null).ToArray(); } private async Task GetAlbumInfoAsync(IEnumerable ids, string market = null, string fields = null) { var url = new Url("/v1/albums/"); url.SetQueryParam("ids", string.Join(",", ids)); if (market != null) url = url.SetQueryParam("market", market); if (fields != null) url = url.SetQueryParam("fields", fields); return await this.GetDataFromSpotify(url); } public async Task GetAllArtistAlbumsAsync(string artistId, string fields = null, string market = null, AlbumType? albumType = null) { return await this.GetByBatches( 50, r => r.total, async batch => await GetArtistAlbumsAsync(artistId, fields, market, albumType, limit: batch.Limit, offset: batch.Offset), r => r.items); } private async Task GetArtistAlbumsAsync(string artistId, string fields = null, string market = null, AlbumType? albumType = null, int? limit = null, int? offset = null) { var url = new Url("/v1/artists/").AppendPathSegments(artistId, "albums"); if (fields != null) url = url.SetQueryParam("fields", fields); if (limit != null) url = url.SetQueryParam("limit", limit); if (offset != null) url = url.SetQueryParam("offset", offset); if (market != null) url = url.SetQueryParam("market", market); if (albumType != null) url = url.SetQueryParam("album_type ", string.Join(",", albumType.Value.GetIndividualFlags().Select(e => e.ToString()))); return await this.GetDataFromSpotify(url); } public async Task GetPublicPlaylistsAsync(string username, string fields = null, int? limit = null, int? offset = null) { var url = new Url("/v1/users/").AppendPathSegments(username, "playlists"); if (fields != null) url = url.SetQueryParam("fields", fields); if (limit != null) url = url.SetQueryParam("limit", limit); if (offset != null) url = url.SetQueryParam("offset", offset); return await this.GetDataFromSpotify(url); } public async Task GetTopTracksAsync(string artistName, int count, string fields = null) { return await this.GetByBatches( 50, r => Math.Min(r.tracks.total, count), async batch => await SearchTracksByArtistAsync(artistName,fields, limit: batch.Limit, offset: batch.Offset), r => r.tracks.items); } public async Task SearchTracksByArtistAsync(string artistName, string fields = null, int? limit = null, int? offset = null) { var url = new Url("https://api.spotify.com/v1/search"); url = url.SetQueryParam("q", string.Format("artist:{0}", artistName)); url = url.SetQueryParam("type", "track"); if (fields != null) url = url.SetQueryParam("fields", fields); if (limit != null) url = url.SetQueryParam("limit", limit); if (offset != null) url = url.SetQueryParam("offset", offset); return await this.GetDataFromSpotify(url); } public async Task SearchTracksAsync(string query, string fields = null, int? limit = null, int? offset = null) { var url = new Url("https://api.spotify.com/v1/search"); url = url.SetQueryParam("q", query); url = url.SetQueryParam("type", "track"); if (fields != null) url = url.SetQueryParam("fields", fields); if (limit != null) url = url.SetQueryParam("limit", limit); if (offset != null) url = url.SetQueryParam("offset", offset); var response = await this.GetDataFromSpotify(url); if (response == null) //The api returns 404 if no results which feels wrong... { return new SearchTracksResponse() { tracks = new SearchTracksCollection() }; } return response; } public async Task SearchArtistsAsync(string artistName, string fields = null, int? limit = null, int? offset = null) { var url = new Url("https://api.spotify.com/v1/search"); url = url.SetQueryParam("q", artistName); url = url.SetQueryParam("type", "artist"); if (fields != null) url = url.SetQueryParam("fields", fields); if (limit != null) url = url.SetQueryParam("limit", limit); if (offset != null) url = url.SetQueryParam("offset", offset); return await this.GetDataFromSpotify(url); } public async Task SearchAllAsync(string searchTerm, List searchTypes, string market = null, string fields = null, int? limit = null, int? offset = null) { var searchTypeQueryParameter = "album,artist,track,playlist"; if (searchTypes != null && searchTypes.Any()) searchTypeQueryParameter = string.Join(",", searchTypes); var url = new Url("https://api.spotify.com/v1/search"); url = url.SetQueryParam("q", searchTerm); url = url.SetQueryParam("type", searchTypeQueryParameter); if (market != null) url = url.SetQueryParam("market", market); if (fields != null) url = url.SetQueryParam("fields", fields); if (limit != null) url = url.SetQueryParam("limit", limit); if (offset != null) url = url.SetQueryParam("offset", offset); return await this.GetDataFromSpotify(url); } public async Task GetUserAsync(string spotifyUsername, string fields = null) { var url = new Url("https://api.spotify.com/v1/users/"); url = url.AppendPathSegment(spotifyUsername); if (fields != null) url = url.SetQueryParam("fields", fields); return await this.GetDataFromSpotify(url); } public async Task GetBrowseFeaturedPlaylists(string country = null) { var url = new Url("https://api.spotify.com/v1/browse/featured-playlists"); if (!string.IsNullOrWhiteSpace(country)) url = url.SetQueryParam("country", country); url = url.SetQueryParam("limit", 50); // 50 is max as specified in the API documentation 2015-02-17 return await this.GetDataFromSpotify(url); } public async Task GetAllBrowseCategoriesAsync(string region = null) { return await this.GetByBatches( 50, r => r.categories.total, async batch => await GetBrowseCategoriesAsync(region, limit: batch.Limit, offset: batch.Offset), r => r.categories.items); } public async Task GetBrowseCategoriesAsync(string country = null, int? limit = null, int? offset = null) { var url = new Url("https://api.spotify.com/v1/browse/categories"); if (!string.IsNullOrWhiteSpace(country)) { url = url.SetQueryParam("country", country); } if (limit != null) { url = url.SetQueryParam("limit", limit); } if (offset != null) { url = url.SetQueryParam("offset", offset); } return await this.GetDataFromSpotify(url); } public async Task GetAllBrowseCategoryPlaylistsAsync(string categoryId, string region = null) { return await this.GetByBatches( 50, r => r.playlists.total, async batch => await GetBrowseCategoryPlaylistsAsync(categoryId, region, limit: batch.Limit, offset: batch.Offset), r => r.playlists.items); } public async Task GetBrowseCategoryPlaylistsAsync(string categoryId, string country = null, int? limit = null, int? offset = null) { var requestUrl = $"https://api.spotify.com/v1/browse/categories/{categoryId}/playlists"; var url = new Url(requestUrl); if (!string.IsNullOrWhiteSpace(country)) url = url.SetQueryParam("country", country); if (limit != null) url = url.SetQueryParam("limit", limit); if (offset != null) url = url.SetQueryParam("offset", offset); return await this.GetDataFromSpotify(url); } public async Task> GetAllAudioFeatureForTracksAsync(IEnumerable trackIds) { var responses = await this.GetBatches(trackIds.Batch(100), GetAudioFeatureForTracksAsync); return responses .Where(r => r.audio_features != null && r.audio_features.Any()) .SelectMany(r => r.audio_features) .ToArray(); } protected async Task GetAudioFeatureForTracksAsync(IEnumerable trackIds) { var requestUrl = $"https://api.spotify.com/v1/audio-features"; var url = new Url(requestUrl); url = url.SetQueryParam("ids", string.Join(",", trackIds)); return await this.GetDataFromSpotifyWithParser(url, async r => { var byteArray = await r.Content.ReadAsByteArrayAsync(); var responseString = Encoding.UTF8.GetString(byteArray, 0, byteArray.Length); return JsonConvert.DeserializeObject(responseString); }); } public AuthenticatedSpotifyWebApi GetAuthenticatedSession(ServiceAccount serviceAccount, IServiceAccountManager serviceAccountManager) { return new AuthenticatedSpotifyWebApi(serviceAccount, _clientId, _clientSecret, serviceAccountManager); } public static AuthenticatedSpotifyWebApi GetAuthenticatedSession( ServiceAccount serviceAccount, IServiceAccountManager serviceAccountManager, string clientId, string clientSecret) { return new AuthenticatedSpotifyWebApi(serviceAccount, clientId, clientSecret, serviceAccountManager); } private async Task GetDataFromSpotify(string url) where T : class { return await this.GetDataFromSpotifyWithParser(url, async r => JsonConvert.DeserializeObject(await r.Content.ReadAsStringAsync())); } private async Task GetDataFromSpotifyWithParser(string url, Func> parser) where T : class { var response = await this.GetDefaultClient().GetAsync(url); if (response.StatusCode == HttpStatusCode.NotFound) { return null; } if (response.IsSuccessStatusCode) { return await parser(response); } throw SpotifyWebAPIException.FromHttpResponseMessage(response); } private async Task GetBatches(IEnumerable input, Func> spotifyFunc) { //var bag = new ConcurrentBag(); //var block = new ActionBlock(async batch => //{ // TResponse response = await spotifyFunc(batch); // bag.Add(response); //}, new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 3 }); //foreach (var batch in input) //{ // await block.SendAsync(batch); //} //block.Complete(); //await block.Completion; //return bag.ToArray(); var tasks = input.Select(async i => await spotifyFunc(i)).ToList(); await tasks.EnsureAllSuccessAsync(); return tasks.Select(t => t.Result).ToArray(); } private async Task GetByBatches( int batchSize, Func getTotalFunc, Func> spotifyFunc, Func> getItemsFunc) { var initialResponse = await spotifyFunc(new Batch(0, batchSize)); if (initialResponse == null) { return Array.Empty(); } var result = new List(getItemsFunc(initialResponse)); if (getTotalFunc(initialResponse) > result.Count) { var responses = await this.GetBatchResponses( getTotalFunc(initialResponse).Batch(batchSize).Skip(1), async batch => { var r = await spotifyFunc(batch); if (r == null) { return Array.Empty(); } return getItemsFunc(r); }); result.AddRange(responses); } return result.ToArray(); } private async Task GetBatchResponses( IEnumerable batches, Func>> spotifyFunc) { //var bag = new ConcurrentBag>(); //var block = new ActionBlock(async batch => //{ // try // { // IEnumerable response = await spotifyFunc(batch); // bag.Add(BatchResponse.CreateSuccess(response, batch)); // } // catch (Exception ex) // { // bag.Add(BatchResponse.CreateError(ex, batch)); // } //}, new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 3 }); //foreach (var batch in batches) //{ // await block.SendAsync(batch); //} //block.Complete(); //await block.Completion; //if (bag.Any(i => i.Success == false)) //{ // throw new SpotifyWebAPIException($"Not all batches succeeded", bag.First(i => i.Success == false).Exception); //} //return bag.OrderBy(r => r.Batch.Offset).SelectMany(r => r.Items).ToArray(); var tasks = batches.Select(async b => { var response = await spotifyFunc(b); return BatchResponse.CreateSuccess(response, b); }).ToArray(); await tasks.EnsureAllSuccessAsync(); return tasks .Select(t => t.Result) .OrderBy(r => r.Batch.Offset) .SelectMany(r => r.Items) .Where(r => r != null) .ToArray(); } private class BatchResponse { public readonly TResponse[] Items; public readonly Batch Batch; public readonly Exception Exception; public bool Success { get { return this.Exception == null; } } private BatchResponse(IEnumerable items, Batch batch, Exception ex) { this.Items = items.ToArray(); this.Exception = ex; this.Batch = batch; } public static BatchResponse CreateSuccess(IEnumerable items, Batch batch) { return new BatchResponse(items, batch, null); } public static BatchResponse CreateError(Exception ex, Batch batch) { return new BatchResponse(new TResponse[0], batch, ex); } } } }