using System; using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; using MoreLinq; using NLog; using Sentry; using Sony.Filtr.ApolloAPI; using Sony.Filtr.ApolloAPI.Models.Apple; using Sony.Filtr.AppleMusic; using Sony.Filtr.AppleMusic.Data; using Sony.Filtr.AppleMusic.Models; using Sony.Filtr.Core.ResponseCollector; using Sony.Filtr.Database; using Sony.Filtr.Tasks.Tasks.AppleMusic.Aggregation; using Sony.Filtr.Utility.Extensions; namespace Sony.Filtr.Tasks.Tasks.AppleMusic { public class FetchAppleMusicChartPlaylistTracks : IScheduledTask { private readonly AppleMusicChartManager _chartManager; private readonly IApolloAppleWebApi _appleMusicApi; private readonly ChartsAggregation _chartsAggregation; private readonly ResponseCollector _responseCollector; private readonly Logger _logger; public FetchAppleMusicChartPlaylistTracks(AppleMusicChartManager chartManager, IApolloAppleWebApi appleMusicApi, ChartsAggregation chartsAggregation, ResponseCollector responseCollector) { _chartManager = chartManager; _appleMusicApi = appleMusicApi; _chartsAggregation = chartsAggregation; _responseCollector = responseCollector; _logger = LogManager.GetLogger("AppleMusicChartPlaylistTracks"); } public async Task ExecuteAsync(Guid scheduledTaskLogId) { const string trackChartName = "track_chart_playlist"; using (SentrySdk.Init("https://4a68838024414276ba558e6b61ee6716@sentry.delphiplatform.io/12")) { var scheduledTaskLog = new ScheduledTaskLog(); var playlists = await _chartManager.GetChartPlaylistIdsPerStorefrontAsync(); var latestPlaylistCharts = _chartManager.GetLatestCharts(new ChartsFilter { ChartType = ChartType.PlaylistBased, }); await playlists.ForEachAsync(3, //foreach (var chartPlaylist in playlists) async chartPlaylist => { using (var conn = await DatabaseHandler.GetOpenConnectionAsync()) { try { var playlistChartSongsData = await GetPlaylistChartSongsAsync(chartPlaylist.TracklistStorefront, chartPlaylist.PlaylistId); if (playlistChartSongsData.HasData == false) { _logger.Error($"Could not get tracks for {chartPlaylist.PlaylistId} in {chartPlaylist.DisplayStorefront}"); return; } _logger.Info($"Fetched amount of songs from AppleApi: {playlistChartSongsData.ChartSongs.Count()} for storefront {chartPlaylist.TracklistStorefront}."); if (this.DbContainsLatestData(latestPlaylistCharts, playlistChartSongsData)) { return; } DateTime chartDate = CreateChartDate(playlistChartSongsData.PlaylistLastModifiedDate); //since we store charts in DB with 1 day shift ahead chartDate = chartDate.AddDays(1); if (_chartManager.GetChart(chartPlaylist.DisplayStorefront, ChartType.PlaylistBased, trackChartName, null, chartDate.Date) != null) { _logger.Error($"Attempt to add chart more than once. Storefront: {chartPlaylist.DisplayStorefront}. ChartType: {ChartType.PlaylistBased}. TrackName: {trackChartName}, GenreId: NULL, ChartDate: {chartDate.Date}"); return; } _logger.Info($"{chartPlaylist.TracklistStorefront} contains new data."); using (var tran = await conn.BeginTransactionAsync()) { try { var chart = await _chartManager.SaveChartAsync(conn, chartPlaylist.DisplayStorefront, null, ChartType.PlaylistBased, trackChartName, chartDate); await _chartManager.SaveSongChartAsync(conn, tran, chart, playlistChartSongsData.ChartSongs); //TODO: When this is activated, make sure to de-activate most played charts on regular apple music charts job (FetchAppleMusicCharts). await _chartManager.SaveCurrentMostPlayedSongChartAsync(chartPlaylist.DisplayStorefront, chartDate, playlistChartSongsData.ChartSongs); _logger.Log(LogLevel.Info, $"Saved MostPlayedSongChart for {chartPlaylist.DisplayStorefront}. Amount of tracks: {playlistChartSongsData.ChartSongs.Count()}"); tran.Commit(); _logger.Info($"Saved playlist based track chart for {chartPlaylist.PlaylistId} in {chartPlaylist.DisplayStorefront}."); } catch (Exception ex) { _logger.Error(ex, $"Could not save new chart data for storefront {chartPlaylist.TracklistStorefront}"); return; } } } catch (Exception ex) { scheduledTaskLog.Error = true; scheduledTaskLog.ErrorMessage = ex.Message; _logger.Error(ex, $"Error in chart playlist for {chartPlaylist.PlaylistId} in {chartPlaylist.DisplayStorefront}. Amount: {playlists.Count()}"); } } }); await SaveTrackSummaryAggregationAsync(); _responseCollector.UploadFile("ResponseData/ChartsResponse.json", "FetchAppleMusicChartPlaylistTracks"); return new ScheduledTaskLog { Finished = DateTimeOffset.UtcNow }; } } private static DateTime CreateChartDate(DateTime appleDate) { DateTime now = DateTime.Now; return new DateTime(appleDate.Year, appleDate.Month, appleDate.Day, now.Hour, now.Minute, now.Second); } private bool DbContainsLatestData(IEnumerable dbCharts, ApplePlaylistListChartSongsData applePlaylistData) { var dbChart = dbCharts.FirstOrDefault( c => c.Storefront.Equals(applePlaylistData.Storefront, StringComparison.InvariantCultureIgnoreCase) && c.Date >= applePlaylistData.PlaylistLastModifiedDate.Date); if (dbChart == null) { return false; } if (applePlaylistData.PlaylistLastModifiedDate > dbChart.Timestamp) { return false; } var existingChartSongs = _chartManager.GetSongChartEntries(dbChart.Id); if (existingChartSongs.Count() != applePlaylistData.ChartSongs.Count()) { return false; } for (int i = 0; i < existingChartSongs.Count; ++i) { var dbSong = existingChartSongs[i]; var appleSong = applePlaylistData.ChartSongs[i]; if (dbSong.ISRC != appleSong.attributes.isrc) { if (dbSong.SongId != appleSong.id) { return false; } } } return true; } private async Task GetPlaylistChartSongsAsync(string playlistStorefront, string playlistId) { var playlist = await _appleMusicApi.GetPlaylistAsync(playlistStorefront, playlistId, "FetchAppleMusicChartPlaylistTracks"); return new ApplePlaylistListChartSongsData(playlist, playlistStorefront); } private async Task SaveTrackSummaryAggregationAsync() { _logger.Debug($"Getting all track positions"); _logger.Debug($"Building chart track summaries"); var availablePlaylistCharts = await _chartManager.GetAvailableChartsOverviewAsync(ChartType.PlaylistBased, "track_chart_playlist"); var availableOldCharts = await _chartManager.GetAvailableChartsOverviewAsync(ChartType.Songs, "most-played"); var joinedAvailableCharts = availablePlaylistCharts.Union(availableOldCharts).GroupBy(a => a.Storefront).Select(overviews => new AvailableSongChartOverview() { Storefront = overviews.First().Storefront, LatestDate = overviews.Select(p => p.LatestDate).Max(), LatestTimestamp = overviews.Select(p => p.LatestTimestamp).Max(), PreviousDate = overviews.Select(p => p.PreviousDate).Max(), PreviousTimestamp = overviews.Select(p => p.PreviousTimestamp).Max() }); var currentCharts = joinedAvailableCharts.ToDictionary(p => p.Storefront); var trackPositions = await GetTrackPositionsAsync(); var trackChartSummary = await _chartsAggregation.BuildTrackChartSummariesAsync(trackPositions, currentCharts); _logger.Debug($"Done building track summaries. Amount: {trackChartSummary.Count()}"); _logger.Debug($"Saving chart track summaries to database. Amount: {trackChartSummary.Count()}"); await _chartsAggregation.SaveTrackChartSummariesAsync(trackChartSummary); _logger.Debug($"Done saving chart track summaries to database"); } private async Task> GetTrackPositionsAsync() { var allPlaylistPosition = await _chartsAggregation.GetAllPlaylistChartPositionsPerTrackAsync(); var allOldChartPosition = await _chartsAggregation.GetAllChartPositionsPerTrackAsync(ChartType.Songs, "most-played"); List chartPositions = allPlaylistPosition; var cappedOldChartPositions = allOldChartPosition.Where(p => p.Position < 100); var playlistData = allPlaylistPosition.Select(p => new { p.Storefront, p.Date }).Distinct().ToHashSet(); var cappedExceptPlaylistPositions = cappedOldChartPositions.Where(p => !playlistData.Contains(new { p.Storefront, p.Date })); chartPositions.AddRange(cappedExceptPlaylistPositions); return chartPositions; } } }