using System; using System.Collections.Concurrent; using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; using NLog; using Sentry; using Sony.Filtr.ApolloAPI; using Sony.Filtr.ApolloAPI.Models.Apple; using Sony.Filtr.AppleMusic; using Sony.Filtr.AppleMusic.Data; using Sony.Filtr.AppleMusic.Playlists; using Sony.Filtr.Core.ResponseCollector; using Sony.Filtr.Database; using Sony.Filtr.Tasks.EventSource; using Sony.Filtr.Tasks.Tasks.AppleMusic.Aggregation; using Sony.Filtr.Utility.Extensions; namespace Sony.Filtr.Tasks.Tasks.AppleMusic { public class FetchAppleMusicCharts : IScheduledTask { private readonly IApolloAppleWebApi _appleMusicApi; private readonly AppleMusicChartManager _chartsManager; private readonly AppleMusicPlaylistManager _appleMusicPlaylistManager; private readonly ResponseCollector _responseCollector; private readonly Logger _logger; public FetchAppleMusicCharts(IApolloAppleWebApi appleMusicApi, AppleMusicChartManager chartsManager, ChartsAggregation chartsAggregation, AppleMusicPlaylistManager appleMusicPlaylistManager, ResponseCollector responseCollector) { _appleMusicApi = appleMusicApi; _chartsManager = chartsManager; _appleMusicPlaylistManager = appleMusicPlaylistManager; _logger = LogManager.GetLogger("ImportAppleMusicData"); _responseCollector = responseCollector; } public async Task ExecuteAsync(Guid scheduledTaskLogId) { using (SentrySdk.Init("https://a5ad71a7fef6428088b82de2ad0f51a9@sentry.delphiplatform.io/10")) { try { _logger.Debug("Getting storefronts"); var storefronts = await _appleMusicApi.GetAllStorefrontsAsync(); using (var duration = new DisposableDuration(() => ApolloFetchAppleMusicChartsEventSource.Log.ProcessStart(storefronts.data.Count), ApolloFetchAppleMusicChartsEventSource.Log.ProcessStop)) { var today = DateTime.UtcNow.Date; List existingCharts; existingCharts = _chartsManager.GetCharts(today); var storefrontIds = storefronts.data.Select(p => p.id).ToList(); _logger.Debug($"Got {storefrontIds.Count} storefronts"); _logger.Debug($"Begin importCharts"); var allCharts = await FetchChartsAsync(storefrontIds, existingCharts); _logger.Debug($"Done importCharts"); await SaveCurrentChartSongToplistAsync(storefrontIds, today, allCharts); _responseCollector.UploadFile("ResponseData/ChartsResponse.json", "FetchAppleMusicCharts"); } } catch (Exception ex) { _logger.Error(ex); ApolloFetchAppleMusicChartsEventSource.Log.ProcessException(ex); } return null; } } private async Task SaveCurrentChartSongToplistAsync(List storefrontIds, DateTime today, ConcurrentBag allCharts) { using (var totalDuration = new DisposableDuration(() => ApolloFetchAppleMusicChartsEventSource.Log.SaveCurrentChartSongTopListStart(storefrontIds.Count, allCharts.Count), ApolloFetchAppleMusicChartsEventSource.Log.SaveCurrentChartSongTopListStop)) { await storefrontIds.ForEachAsync(5, async storefront => { try { using (var saveCurrentDuration = new DisposableDuration(() => ApolloFetchAppleMusicChartsEventSource.Log.SaveCurrentMostPlayedSongChartStart(storefront), ApolloFetchAppleMusicChartsEventSource.Log.SaveCurrentMostPlayedSongChartStop)) { var mostPlayedSongChart = allCharts.SingleOrDefault(c => c.Storefront == storefront && !c.GenreId.HasValue)?.ChartData.Songs.GetValueOrDefault("most-played"); if (mostPlayedSongChart == null) { _logger.Debug($"Fetching Most played song chart for {storefront}"); mostPlayedSongChart = await _appleMusicApi.GetPaginatedSongChartAsync(storefront, chartname: "most-played", genreId: null); ApolloFetchAppleMusicChartsEventSource.Log.RequestedAppleMusicSongChart(storefront, mostPlayedSongChart.Count); } if (mostPlayedSongChart.Any()) { _logger.Debug($"Storing Most played song chart for {storefront}, Count: {mostPlayedSongChart.Count()}"); await _chartsManager.SaveCurrentMostPlayedSongChartAsync(storefront, today, mostPlayedSongChart); ApolloFetchAppleMusicChartsEventSource.Log.SavedAppleMusicSongChart(storefront, mostPlayedSongChart.Count); } else { _logger.Warn($"Could not get Most played song chart for {storefront}"); ApolloFetchAppleMusicChartsEventSource.Log.NoMostPlayedChart(storefront); } } } catch (Exception ex) { _logger.Error(ex); ApolloFetchAppleMusicChartsEventSource.Log.SaveCurrentMostPlayedSongChartException(ex); } }); } } private async Task SaveChartAsync(ChartDataGroup fullChartViewModel) { ApolloFetchAppleMusicChartsEventSource.Log.SaveChartStart(fullChartViewModel.Storefront, fullChartViewModel.GenreId, fullChartViewModel.ChartData.Albums.Values.Count(v => v.Any()), fullChartViewModel.ChartData.MusicVideos.Values.Count(v => v.Any()), fullChartViewModel.ChartData.Playlists.Values.Count(v => v.Any()), fullChartViewModel.ChartData.Songs.Values.Count(v => v.Any())); using (var conn = await DatabaseHandler.GetOpenConnectionAsync()) { using (var tran = await conn.BeginTransactionAsync()) { var songs = fullChartViewModel.ChartData?.Songs; if (songs != null) { foreach (var songChartList in songs.Where(s => s.Value.Any())) { _logger.Log(LogLevel.Info, $"Saving to db song chart: {fullChartViewModel.Storefront}, {fullChartViewModel.GenreId}, {ChartType.Songs}, {songChartList.Key}."); var songsChartId = await _chartsManager.SaveChartAsync(conn, fullChartViewModel.Storefront, fullChartViewModel.GenreId, ChartType.Songs, songChartList.Key); _logger.Log(LogLevel.Info, $"Saving to db songChartList: {fullChartViewModel.Storefront}, {fullChartViewModel.GenreId}, {ChartType.Songs}, {songChartList.Key}, {songChartList.Value.Count()}."); await _chartsManager.SaveSongChartAsync(conn, tran, songsChartId, songChartList.Value); } } var albums = fullChartViewModel.ChartData?.Albums; if (albums != null) { foreach (var albumChartList in albums.Where(a => a.Value.Any())) { _logger.Log(LogLevel.Info, $"Saving to db album chart: {fullChartViewModel.Storefront}, {fullChartViewModel.GenreId}, {ChartType.Songs}, {albumChartList.Key}."); var songsChartId = await _chartsManager.SaveChartAsync(conn, fullChartViewModel.Storefront, fullChartViewModel.GenreId, ChartType.Albums, albumChartList.Key); _logger.Log(LogLevel.Info, $"Saving to db albumChartList: {fullChartViewModel.Storefront}, {fullChartViewModel.GenreId}, {ChartType.Songs}, {albumChartList.Key}, {albumChartList.Value.Count()}."); await _chartsManager.SaveAlbumChartAsync(conn, tran, songsChartId, albumChartList.Value); } } var musicVideos = fullChartViewModel.ChartData?.MusicVideos; if (musicVideos != null) { foreach (var musicVideoChartList in musicVideos.Where(m => m.Value.Any())) { if (musicVideoChartList.Value.Any()) { _logger.Log(LogLevel.Info, $"Saving to db music video chart: {fullChartViewModel.Storefront}, {fullChartViewModel.GenreId}, {ChartType.Songs}, {musicVideoChartList.Key}."); var songsChartId = await _chartsManager.SaveChartAsync(conn, fullChartViewModel.Storefront, fullChartViewModel.GenreId, ChartType.MusicVideos, musicVideoChartList.Key); _logger.Log(LogLevel.Info, $"Saving to db musicVideoChartList: {fullChartViewModel.Storefront}, {fullChartViewModel.GenreId}, {ChartType.Songs}, {musicVideoChartList.Key}, {musicVideoChartList.Value.Count()}."); await _chartsManager.SaveMusicVideoChartAsync(conn, tran, songsChartId, musicVideoChartList.Value); } } } var playlists = fullChartViewModel.ChartData?.Playlists; if (playlists != null) { foreach (var playlistList in playlists.Where(p => p.Value.Any())) { var playlistIds = playlistList.Value.Select(x => x.id).ToList(); var playlistsByIds = await _appleMusicPlaylistManager.GetPlaylistsByIds(playlistIds); if (playlistIds.Count() != playlistsByIds.Count) { var ids = playlistIds.Except(playlistsByIds.Select(x => x.Id)); foreach (var id in ids) { var pl = playlistList.Value.Where(x => x.id == id).FirstOrDefault(); var curator = _chartsManager.GetCuratorByName(pl.attributes.curatorName).FirstOrDefault(); await _appleMusicPlaylistManager.AddOrUpdatePlaylistAsync(new AppleMusicPlaylist { Id = pl.id, Artwork = pl.attributes.artwork.url, CuratorId = curator?.Id, LatestUpdate = pl.attributes.lastModifiedDate != null ? DateTime.Parse(pl.attributes.lastModifiedDate) : DateTime.Now, PlaylistType = pl.attributes.playlistType, Name = pl.attributes.name }); } } var playlistChartId = await _chartsManager.SaveChartAsync(conn, fullChartViewModel.Storefront, fullChartViewModel.GenreId, ChartType.Playlists, playlistList.Key); _logger.Log(LogLevel.Info, $"Saving to db playlistChart: {playlistChartId}, {playlistList.Value.Count()}, {fullChartViewModel.Storefront}."); await _chartsManager.SavePlaylistChartAsync(conn, tran, playlistChartId, playlistList.Value, fullChartViewModel.Storefront); } } tran.Commit(); } } ApolloFetchAppleMusicChartsEventSource.Log.SaveChartStop(fullChartViewModel.Storefront); } private async Task> FetchChartsAsync(List storefrontIds, List existingCharts) { ConcurrentBag allCharts = new ConcurrentBag(); using (var fetchChartsDuration = new DisposableDuration(() => ApolloFetchAppleMusicChartsEventSource.Log.FetchChartsStart(storefrontIds.Count), ApolloFetchAppleMusicChartsEventSource.Log.FetchChartsStop)) { await storefrontIds.ForEachAsync(5, async storefrontId => { _logger.Debug($"Getting chart genres for {storefrontId}"); try { var hasStoredChart = existingCharts.Any(c => c.Storefront == storefrontId && !c.GenreId.HasValue); if (!hasStoredChart) { ApolloFetchAppleMusicChartsEventSource.Log.GetFullChartFromAppleMusicStart(storefrontId); _logger.Debug($"Getting global chart for {storefrontId}"); var noGenreChart = await _appleMusicApi.GetFullChartAsync(storefrontId); _logger.Log(LogLevel.Info, $"Fetched from AppleApi for no genre storefront '{storefrontId}'. Songs: {noGenreChart.Songs.Count()}, music videos: {noGenreChart.MusicVideos.Count()}, " + $"albums: {noGenreChart.Albums.Count()}, playlists: {noGenreChart.Playlists.Count()}."); var noGenreDataGroup = new ChartDataGroup() { GenreId = null, Storefront = storefrontId, ChartData = noGenreChart }; await SaveChartAsync(noGenreDataGroup); allCharts.Add(noGenreDataGroup); ApolloFetchAppleMusicChartsEventSource.Log.GetFullChartFromAppleMusicStop(storefrontId); } } catch (Exception ex) { _logger.Error(ex, $"Error getting global chart for {storefrontId}"); } try { var chartGenres = await _appleMusicApi.GetChartGenresAsync(storefrontId); await _chartsManager.SaveGenresAsync(storefrontId, chartGenres); ApolloFetchAppleMusicChartsEventSource.Log.SavedStoreFrontChartGenres(storefrontId, chartGenres.Count); foreach (var chartGenre in chartGenres) { try { var hasStoredChart = existingCharts.Any(c => c.Storefront == storefrontId && c.GenreId == chartGenre.id); if (!hasStoredChart) { _logger.Debug($"Getting chart for {storefrontId} and genre {chartGenre.attributes.name} ({chartGenre.id})"); var genreChart = await _appleMusicApi.GetFullChartAsync(storefrontId, chartGenre.id); _logger.Log(LogLevel.Info, $"Fetched from AppleApi storefront '{storefrontId}' for genre: {chartGenre.id} ({chartGenre.attributes.name}). Songs: {genreChart.Songs.Count()}, music videos: {genreChart.MusicVideos.Count()}, " + $"albums: {genreChart.Albums.Count()}, playlists: {genreChart.Playlists.Count()}."); var genreChartDataGroup = new ChartDataGroup() { GenreId = chartGenre.id, Storefront = storefrontId, ChartData = genreChart }; await SaveChartAsync(genreChartDataGroup); allCharts.Add(genreChartDataGroup); } } catch (Exception ex) { _logger.Error(ex, $"Error getting chart for {storefrontId} and genre {chartGenre.attributes.name} ({chartGenre.id})"); } } } catch (Exception ex) { _logger.Error(ex, $"Error getting genres for {storefrontId}"); } }); } return allCharts; } } public class ChartPosition { public string Storefront { get; set; } public string ISRC { get; set; } public DateTime Date { get; set; } public DateTime TimeStamp { get; set; } public int Position { get; set; } } class ChartDataGroup { public string Storefront { get; set; } public long? GenreId { get; set; } public AppleFullChartViewModel ChartData { get; set; } } }