using MoreLinq; using NLog; using PetaPoco.Business; using Sony.Filtr.ApolloAPI; using Sony.Filtr.AppleMusic.Data; using Sony.Filtr.Utility.Extensions; using System; using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; namespace Sony.Filtr.AppleMusic.Helpers { public class AppleMusicUpdateHelper { private readonly IApolloAppleWebApi _appleMusicApi; private readonly AppleMusicSongManager _appleMusicSongManager; private const int _MaxArtistBatchSize = 25; public AppleMusicUpdateHelper(IApolloAppleWebApi appleMusicApi, AppleMusicSongManager appleMusicSongManager) { _appleMusicApi = appleMusicApi; _appleMusicSongManager = appleMusicSongManager; } public async Task SaveSongsWithRelatedDataAsync(IList songs, Logger logger) { await BatchSaveSongsToDatabaseAsync(songs, logger); await SaveSongRelatedArtistsAsync(songs, logger); await SaveSongArtistConnectionAsync(songs, logger); await SaveSongRelatedAlbumsAsync(songs, logger); } public async Task> FetchAppleMusicSongsAsync(string storefront, IEnumerable songIds) { var songData = await _appleMusicApi.GetSongsAsync(storefront, songIds); return songData.Where(s => s?.attributes != null).Select(s => { var attributes = s.attributes; var artists = s.relationships?.artists?.data; var albums = s.relationships?.albums?.data; return new AppleMusicSong() { Id = s.id, AlbumId = albums?.FirstOrDefault()?.id ?? 0, ArtistId = artists?.FirstOrDefault()?.id ?? 0, ArtistIds = artists?.Select(a => a.id).ToList(), ArtistName = attributes.artistName, ArtworkUrl = attributes.artwork?.url, ComposerName = attributes.composerName, DiscNumber = attributes.discNumber, Duration = attributes.durationInMillis, Name = attributes.name, Storefront = storefront, TrackNumber = attributes.trackNumber, Isrc = attributes.isrc, ReleaseDate = ExtractReleaseDateFromString(attributes.releaseDate), }; }).ToList(); } private async Task BatchSaveSongsToDatabaseAsync(ICollection songs, Logger logger) { if (!songs.Any()) return; logger.Debug($"Saving {songs.Count} songs to database."); var songBatches = songs.Batch(100).ToList(); await songBatches.ForEachAsync(1, async songBatch => { var localAppleMusicSongs = songBatch.ToList(); await _appleMusicSongManager.AddOrUpdateSongsAsync(localAppleMusicSongs); }); } private async Task SaveSongRelatedAlbumsAsync(IEnumerable songs, Logger logger) { foreach (var storefrontSongs in songs.GroupBy(p => p.Storefront)) { var storefront = storefrontSongs.Key; var uniqueAlbums = storefrontSongs.Select(s => s.AlbumId).Distinct().ToList(); logger.Debug($"Start saving albums for {uniqueAlbums.Count} songs in {storefront}."); try { var batches = uniqueAlbums.Batch(25).ToList(); await batches.ForEachAsync(10, async albumIdBatch => { try { var albums = await FetchAppleMusicAlbumsAsync(storefront, albumIdBatch); await PetaPocoRepository.Instance.BulkInsertAsync(albums, ignoreExisting: true); logger.Debug($"Saved {albums.Count} albums."); } catch (Exception ex) { logger.Error(ex, "Could not load batch of albums"); } }); } catch (Exception ex) { logger.Error(ex, $"Could not load albums from {storefront}"); } } } private async Task SaveSongRelatedArtistsAsync(ICollection allSongs, Logger logger) { var uniqueArtists = allSongs.Where(p => p.ArtistIds != null).SelectMany(s => s.ArtistIds, (song, artistId) => new { song.Storefront, ArtistId = artistId }).Distinct().ToList(); logger.Debug($"Identified {uniqueArtists.Count} combinations of storefront and artist."); foreach (var storefrontArtistIds in uniqueArtists.GroupBy(p => p.Storefront)) { var storefront = storefrontArtistIds.Key; var artistIds = storefrontArtistIds.Select(p => p.ArtistId).Distinct().ToList(); logger.Debug($"Getting {artistIds.Count} artists in {storefront}"); var artistBatches = artistIds.Batch(_MaxArtistBatchSize).ToList(); await artistBatches.ForEachAsync(2, async batchArtistIds => { try { var localArtistData = await _appleMusicApi.GetArtistsAsync(storefront, batchArtistIds); var localArtists = localArtistData.Select(a => new AppleMusicArtist() { Storefront = storefront, Id = a.id, Name = a.attributes.name }).ToList(); await _appleMusicSongManager.AddOrUpdateArtistsAsync(storefront, localArtists); } catch (Exception ex) { logger.Error(ex, "Could not load artist-batch"); } }); logger.Debug($"Done saving {artistIds.Count} artists in {storefront}"); } } private async Task SaveSongArtistConnectionAsync(ICollection songs, Logger logger) { foreach (var storefrontSongs in songs.GroupBy(p => p.Storefront)) { var storefront = storefrontSongs.Key; logger.Debug($"Start saving song-artist-connections for {storefrontSongs.Count()} songs in {storefront}."); await storefrontSongs.ToList().ForEachAsync(10, async song => { try { await _appleMusicSongManager.AddOrUpdateSongArtistConnectionsAsync(storefront, song); } catch (Exception ex) { logger.Error(ex, $"Could not save song-artist-connection for {song.Id} in {storefront}"); } }); logger.Debug($"Done saving song-artist-connections for {storefrontSongs.Count()} songs in {storefront}."); } } private async Task> FetchAppleMusicAlbumsAsync(string storefront, IEnumerable albumIdsBatch) { var albumData = await _appleMusicApi.GetAlbumsAsync(storefront, albumIdsBatch); return albumData.Where(a => a?.attributes != null).Select(a => { var attributes = a.attributes; return new AppleMusicAlbum() { Id = a.id, ArtistName = attributes.artistName, ArtworkUrl = attributes.artwork?.url, Storefront = storefront, Name = attributes.name, Copyright = attributes.copyright, IsMasteredForItunes = attributes.isMasteredForItunes, IsSingle = attributes.isSingle, RecordLabel = attributes.recordLabel, ReleaseDate = ExtractReleaseDateFromString(attributes.releaseDate), ReleaseYear = ExtractAlbumReleaseYear(attributes.releaseDate), TrackCount = attributes.trackCount }; }).ToList(); } private static DateTime? ExtractReleaseDateFromString(string attributesReleaseDate) { if (DateTime.TryParse(attributesReleaseDate, out var releaseDate)) return releaseDate; return null; } private int? ExtractAlbumReleaseYear(string attributesReleaseDate) { if (DateTime.TryParse(attributesReleaseDate, out var releaseDate)) return releaseDate.Year; if (int.TryParse(attributesReleaseDate, out var releaseYear)) return releaseYear; return null; } } }