using MoreLinq; using MySql.Data.MySqlClient; using NLog; using Sony.Filtr.AppleMusic; using Sony.Filtr.Database; using Sony.Filtr.Tasks.Tasks.AppleMusic; using Sony.Filtr.Utility.ConsoleUtility; using Sony.Filtr.Utility.Extensions; using System; using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; using Sony.Filtr.AppleMusic.Helpers; namespace Sony.Filtr.Worker.Onetime { public class AppleMusicSongAlbumDataBackfill { private readonly UpdateAppleMusicSongsTask _appleMusicSongsTask; private readonly AppleMusicSongManager _songManager; private readonly AppleMusicUpdateHelper _appleMusicUpdateHelper; private readonly Logger _logger; private const int _MaxSongBatchSize = 300; public AppleMusicSongAlbumDataBackfill(UpdateAppleMusicSongsTask appleMusicSongsTask, AppleMusicSongManager songManager, AppleMusicUpdateHelper appleMusicUpdateHelper) { _appleMusicSongsTask = appleMusicSongsTask; _songManager = songManager; _appleMusicUpdateHelper = appleMusicUpdateHelper; _logger = LogManager.GetLogger("UpdateAppleMusicSongsTask"); } public async Task ExecuteAsync() { //var allSongIds = await _songManager.GetSongsWithoutAlbumAsync(); var allSongIds = await GetAllSongIds(); //var allChartSongs = await GetChartSongIds(); //foreach(var storefrontSongs in allChartSongs) //{ // var storefrontChartSongs = allChartSongs[storefrontSongs.Key]; // var storefrontAllSongs = allSongsIds[storefrontSongs.Key]; // var songsToFetchFor = storefrontChartSongs.Except(storefrontAllSongs); // Console.WriteLine(songsToFetchFor.Count()); //} foreach (var storefront in allSongIds) { //var storefrontSongs = new ConcurrentBag(); try { var batches = storefront.Value.Batch(_MaxSongBatchSize).ToList(); var totalBatches = batches.Count; _logger.Debug($"Fetching {storefront.Value.Count} songs in {storefront.Key}"); using (var progress = new ConsoleProgressBarCount(totalBatches)) { await batches.ForEachAsync(10, async songIdsBatch => { try { var songs = await _appleMusicUpdateHelper.FetchAppleMusicSongsAsync(storefront.Key, songIdsBatch); //_logger.Debug($"Got {songs.Count} songs from API."); await _songManager.AddOrUpdateSongsAsync(songs); } catch (Exception ex) { _logger.Error(ex, "Could not load batch of songs"); } }, progress); } } catch(Exception ex) { _logger.Error(ex, $"Could not load songs from {storefront}"); } //await _appleMusicSongsTask.SaveSongsAsync(storefrontSongs); //await _appleMusicSongsTask.GetAndSaveArtists(storefrontSongs); //await _appleMusicSongsTask.SaveSongArtistConnection(storefrontSongs); //await _appleMusicSongsTask.GetAndSaveAlbumsAsync(storefrontSongs); } //var allSongs = await _appleMusicSongsTask.GetSongsAsync(allSongsIds); } private async Task>> GetAllSongIds() { var songIdsByStorefront = new Dictionary>(StringComparer.InvariantCultureIgnoreCase); using (var conn = await DatabaseHandler.GetOpenReadOnlyConnectionAsync()) { var sql = "SELECT Id, Storefront " + "FROM tblAppleMusicSong AS cs " + "WHERE ReleaseDate IS NULL "; using (var cmd = new MySqlCommand(sql, conn)) { using (var reader = await cmd.ExecuteReaderAsync()) { while (await reader.ReadAsync()) { var songId = reader.GetInt64(0); var storefront = reader.GetString(1); if (!songIdsByStorefront.ContainsKey(storefront)) songIdsByStorefront.Add(storefront, new List()); songIdsByStorefront[storefront].Add(songId); } } } } return songIdsByStorefront; } //private async Task>> GetChartSongIds() //{ // var songIdsByStorefront = new Dictionary>(StringComparer.InvariantCultureIgnoreCase); // using (var conn = await DatabaseHandler.GetOpenReadOnlyConnectionAsync()) // { // var sql = "SELECT DISTINCT cs.SongId, c.Storefront " + // "FROM tblAppleMusicChartSongEntry AS cs " + // "INNER JOIN tblAppleMusicChart AS c ON cs.chartId = c.id"; // using (var cmd = new MySqlCommand(sql, conn)) // { // using (var reader = await cmd.ExecuteReaderAsync()) // { // while (await reader.ReadAsync()) // { // var songId = reader.GetInt64(0); // var storefront = reader.GetString(1); // if (!songIdsByStorefront.ContainsKey(storefront)) // songIdsByStorefront.Add(storefront, new List()); // songIdsByStorefront[storefront].Add(songId); // } // } // } // } // return songIdsByStorefront; //} } }