using MySql.Data.MySqlClient; using NLog; using Sony.Filtr.Buzz; using Sony.Filtr.Contracts.Entities; using Sony.Filtr.Core.TopLists; using Sony.Filtr.Database; using Sony.Filtr.Playlists.Spotify; using Sony.Filtr.Utility.ConsoleUtility; using Sony.Filtr.Utility.Extensions; using System; using System.Collections.Concurrent; using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; namespace Sony.Filtr.Tasks.Tasks.Spotify { public class SpotifyMostPlaylistedTrackTask : IScheduledTask { private readonly BuzzAccountManager _buzzAccountManager; private readonly TopListManager _topListManager; private readonly SpotifyPlaylistManager _spotifyPlaylistManager; private readonly Logger _logger; public SpotifyMostPlaylistedTrackTask(BuzzAccountManager buzzAccountManager, TopListManager topListManager, SpotifyPlaylistManager spotifyPlaylistManager) { _buzzAccountManager = buzzAccountManager; _topListManager = topListManager; _spotifyPlaylistManager = spotifyPlaylistManager; _logger = LogManager.GetLogger("SpotifyMostPlaylistedTrackTask"); } public async Task ExecuteAsync(Guid scheduledTaskLogId) { var taskLog = new ScheduledTaskLog(); var dataLog = new ScheduledTaskDataLog("SpotifyMostPlaylistedTrack", DateTimeOffset.UtcNow, scheduledTaskLogId); var allPlaylistedTracks = await GetAllPlaylistedTracksAsync(); _logger.Debug($"Got all tracks from db. "); int totalIsrcs = allPlaylistedTracks.Count; _logger.Debug($"Total number of ISRCs: {totalIsrcs}."); await UpdateToplistAsync(allPlaylistedTracks); _logger.Debug($"Done storing all isrc totals"); dataLog.Rows = totalIsrcs; dataLog.Finished = DateTimeOffset.UtcNow; taskLog.DataLogs.Add(dataLog); return taskLog; } private async Task UpdateToplistAsync(List grouping) { var filterCombinations = await GetFilterCombinationsAsync(); _logger.Debug($"Got all filter combinations. "); var toplists = BuildNewPlaylistTrackTopLists(filterCombinations, grouping); _logger.Debug($"Done building all toplist combinations"); var toplistDic = new ConcurrentDictionary>(toplists.GroupBy(t => t.CombinationId).ToDictionary(k => k.Key, v => v.ToList())); await filterCombinations.ForEachAsync(5, async combination => { _logger.Debug($"Saving toplist for filter {combination.Id}"); var toplist = toplistDic.GetValueOrDefault(combination.Id); await _topListManager.SetTopListAsync(combination, toplist); }); _logger.Debug($"Done storing all toplists"); var uniqueIsrc = toplists.Select(t => t.Isrc).Distinct(); var isrcPlaylists = grouping.Join(uniqueIsrc, pair => pair.Isrc, s => s, (pair, valuePair) => new { Isrc = pair.Isrc, Playlists = pair.PlaylistReferences.Count }).ToDictionary(k => k.Isrc, v => v.Playlists); await _topListManager.SetTrackTotalPlaylistsAsync(isrcPlaylists); } private ConcurrentBag BuildNewPlaylistTrackTopLists(List filterCombinations, List grouping) { ConcurrentBag toplists = new ConcurrentBag(); foreach(var countryFilterGroup in filterCombinations.GroupBy(f => f.CountryCode)) { _logger.Debug($"Begin calculating toplist for region: {countryFilterGroup.Key}"); var tempValues = grouping.Select(b => new IsrcGroup { Isrc = b.Isrc, Frontline = b.Frontline, PlaylistReferences = b.PlaylistReferences.Where(p => MatchesRegionFiltering(p, countryFilterGroup.Key)).ToList() }).Where(b => b.PlaylistReferences.Any()).ToList(); foreach(var buzzFilterGroup in countryFilterGroup.GroupBy(b => b.BuzzCategoryId)) { var tempValues1 = tempValues.Select(b => new IsrcGroup { Isrc = b.Isrc, Frontline = b.Frontline, PlaylistReferences = b.PlaylistReferences.Where(p => MatchesBuzzCategoryFiltering(p, buzzFilterGroup.Key)).ToList() }).Where(b => b.PlaylistReferences.Any()).ToList(); foreach(var releaseTypeFilterGroup in buzzFilterGroup.GroupBy(f => f.TrackReleaseType)) { var tempValues2 = tempValues1.Select(b => new IsrcGroup { Isrc = b.Isrc, Frontline = b.Frontline, PlaylistReferences = b.PlaylistReferences.Where(p => MatchesReleaseType(b.Frontline, releaseTypeFilterGroup.Key)).ToList() }).Where(b => b.PlaylistReferences.Any()).ToList(); foreach (var filter in releaseTypeFilterGroup) { _logger.Debug($"Calculating toplist for combination: startDate: {filter.TimeSpanType}, buzzCategory: {filter.BuzzCategoryId}, region: {filter.CountryCode}, trackReleaseType: {filter.TrackReleaseType}"); var toplist = tempValues2.Select(p => new PlaylistTrackTopListEntry() { Isrc = p.Isrc, CombinationId = filter.Id, CountInFilter = p.PlaylistReferences.Count(pi => MatchesDateFiltering(pi, filter.TimeSpanType)), }).Where(c => c.CountInFilter > 0).OrderByDescending(p => p.CountInFilter).Take(250).ToList(); toplist.ForEach(t => toplists.Add(t)); } } } } return toplists; } private TopListType CalculateEarliestAdded(PlaylistedTrack playlistedTrack, Dictionary dates) { TopListType type = TopListType.TopPlaylists; foreach (var dateTime in dates) { if (playlistedTrack.EarliestAdded.HasValue && playlistedTrack.EarliestAdded.Value.Date >= dateTime.Value) { type = type | dateTime.Key; } } return type; } private Dictionary CalculateToplistEarliestAddedDates() { Dictionary earliestAddedDates = new Dictionary(); earliestAddedDates.Add(TopListType.TopPlaylists, null); earliestAddedDates.Add(TopListType.TopPlaylistsAdded3Days, DateTime.Today.AddDays(-3)); earliestAddedDates.Add(TopListType.TopPlaylistsAdded7Days, DateTime.Today.AddDays(-7)); earliestAddedDates.Add(TopListType.TopPlaylistsAdded1Month, DateTime.Today.AddMonths(-1)); earliestAddedDates.Add(TopListType.TopPlaylistsAdded3Month, DateTime.Today.AddMonths(-3)); return earliestAddedDates; } private async Task> GetFilterCombinationsAsync() { List toplistTypes = new List { TopListType.TopPlaylists, TopListType.TopPlaylistsAdded3Days, TopListType.TopPlaylistsAdded7Days, TopListType.TopPlaylistsAdded1Month, TopListType.TopPlaylistsAdded3Month }; var buzzCategories = _buzzAccountManager.GetBuzzCategories(); List buzzCategoryIds = new List { null }; buzzCategoryIds.AddRange(buzzCategories.Select(p => (int?)p.ID)); var buzzUsers = await _buzzAccountManager.GetBuzzUsersAsync(); var regions = buzzUsers.Select(p => p.CountryCode).FilterNull().Distinct(StringComparer.InvariantCultureIgnoreCase); List regionCodes = new List { null }; regionCodes.AddRange(regions); List trackReleaseTypes = new List { TrackReleaseType.Both, TrackReleaseType.Frontline, TrackReleaseType.Catalog }; var existingCombinations = _topListManager.GetFilteringCombinations(); List allCombinations = new List(); foreach (var toplistType in toplistTypes) { foreach (var buzzCategoryId in buzzCategoryIds) { foreach (var region in regionCodes) { foreach (var trackReleaseType in trackReleaseTypes) { var filteringCombination = existingCombinations.SingleOrDefault(p => p.TimeSpanType == toplistType && p.BuzzCategoryId == buzzCategoryId && p.CountryCode == region && p.TrackReleaseType == trackReleaseType); if (filteringCombination == null) { //Add combination to db. filteringCombination = _topListManager.AddFilteringCombination(new PlaylistTrackFilterCombination() { TimeSpanType = toplistType, BuzzCategoryId = buzzCategoryId, CountryCode = region, TrackReleaseType = trackReleaseType, }); } allCombinations.Add(filteringCombination); } } } } return allCombinations; } private async Task> GetAllPlaylistedTracksAsync() { ConcurrentBag allPlaylistedTracks = new ConcurrentBag(); _logger.Debug($"Begin GetAllPlaylistedTracksAsync"); var playlists = _spotifyPlaylistManager.GetAllPlaylists(); _logger.Debug($"Found {playlists.Count} playlists"); using (var progress = new ConsoleProgressBarCount(playlists.Count)) { await playlists.ForEachAsync(10, async playlist => { await FaultHandlingPolicy.MySqlRetryPolicyAsync.ExecuteAsync(async () => { var tracksForPlaylist = await GetPlaylistedTracksAsync(playlist); tracksForPlaylist.ForEach(allPlaylistedTracks.Add); }); }, progress); } _logger.Debug($"Done reading track data from {playlists.Count} playlists"); var dates = CalculateToplistEarliestAddedDates(); var grouping = allPlaylistedTracks.GroupBy(p => p.Isrc, (isrc, tracks) => { var indexedCollection = tracks.Select(t => new FilterPlaylistReferences() { CountryCode = t.CountryCode, BuzzCategoryId = t.BuzzCategoryId, TopListTypes = CalculateEarliestAdded(t, dates), EarliestAdded = t.EarliestAdded }).ToList(); return new IsrcGroup() { Isrc = isrc, Frontline = ISRC.Parse(isrc)?.IsFrontline ?? false, PlaylistReferences = indexedCollection }; }, StringComparer.InvariantCultureIgnoreCase).ToList(); return grouping; } private async Task> GetPlaylistedTracksAsync(SpotifyPlaylist playlist) { List playlistedTracks = new List(); using (var conn = await DatabaseHandler.GetOpenReadOnlyConnectionAsync()) { const string sql = "SELECT DISTINCT t.ISRC, pt.EarliestAdded" + " FROM tblSpotifyPlaylistTrackList2 AS pt " + " INNER JOIN tblSpotifyTrack2 AS t ON pt.TrackId = t.TrackId " + " WHERE pt.PlaylistId = @playlistId AND t.ISRC IS NOT NULL"; var cmd = new MySqlCommand(sql, conn); cmd.Parameters.AddWithValue("@playlistId", playlist.PlaylistId); var reader = await cmd.ExecuteReaderAsync(); while(await reader.ReadAsync()) { var playlistedTrack = new PlaylistedTrack() { Isrc = reader.GetString(0), EarliestAdded = reader.IsDBNull(1) ? (DateTime?)null : reader.GetDateTime(1), BuzzCategoryId = playlist.BuzzCategoryId, CountryCode = playlist.CountryCode, }; playlistedTracks.Add(playlistedTrack); } } return playlistedTracks; } private static bool MatchesReleaseType(bool isFrontline, TrackReleaseType trackReleaseType) { if (trackReleaseType == TrackReleaseType.Both) return true; if (trackReleaseType == TrackReleaseType.Frontline) return isFrontline; if (trackReleaseType == TrackReleaseType.Catalog) return !isFrontline; return false; } private static bool MatchesRegionFiltering(FilterPlaylistReferences pi, string region) { if (region == null) return true; return region == pi.CountryCode; } private static bool MatchesBuzzCategoryFiltering(FilterPlaylistReferences pi, int? buzzCategoryId) { if (!buzzCategoryId.HasValue) return true; return pi.BuzzCategoryId == buzzCategoryId; } private static bool MatchesDateFiltering(FilterPlaylistReferences pi, TopListType topListType) { return pi.TopListTypes.HasFlag(topListType); } private class PlaylistedTrack { public string Isrc { get; set; } public DateTime? EarliestAdded { get; set; } public int? BuzzCategoryId { get; set; } public string CountryCode { get; set; } } private class FilterPlaylistReferences { public TopListType TopListTypes { get; set; } public DateTime? EarliestAdded { get; set; } public int? BuzzCategoryId { get; set; } public string CountryCode { get; set; } } private class IsrcGroup { public string Isrc { get; set; } public bool Frontline { get; set; } public List PlaylistReferences { get; set; } } } }