using System; using System.Collections.Generic; using System.Threading.Tasks; using System.Threading.Tasks.Dataflow; using MoreLinq; using MySql.Data.MySqlClient; using NLog; using Sony.Filtr.Database; using Sony.Filtr.ErrorLogging; using Sony.Filtr.Playlists.Spotify; using Sony.Filtr.SpotifyImages; using Sony.Filtr.SpotifyWebAPI; using Sony.Filtr.SpotifyWebAPI.Model; using Sony.Filtr.Utility.Extensions; namespace Sony.Filtr.Tasks.Tasks.Spotify { public class SpotifyAlbumDataTask : IScheduledTask { private readonly SpotifyAlbumManager _spotifyAlbumManager; private readonly SpotifyWebApi _spotifyWebApi; private readonly SpotifyImageDownloader _spotifyImageHandler; private Logger _logger; public SpotifyAlbumDataTask(SpotifyAlbumManager spotifyAlbumManager, SpotifyWebApi spotifyWebApi, SpotifyImageDownloader spotifyImageHandler) { _spotifyAlbumManager = spotifyAlbumManager; _spotifyWebApi = spotifyWebApi; _spotifyImageHandler = spotifyImageHandler; _logger = LogManager.GetLogger("SpotifyAlbumDataTask"); } public async Task ExecuteAsync(Guid scheduledTaskLogId) { await SaveSpotifyAlbumImagesAsync(); await SetSpotifyTrackPropertiesAsync(); return null; } private async Task SaveSpotifyAlbumImagesAsync() { var getAlbumInfoBlock = new TransformManyBlock, FullAlbumItem>(async batch => { Func, Task>> safeLoadDataFromSpotify = async b => { try { return await _spotifyWebApi.GetAllAlbumInfoAsync(b); } catch (Exception ex) { _logger.Error(ex, "Could not load data from Spotify"); return null; } }; var result = await safeLoadDataFromSpotify(batch); if (result == null) { result = await safeLoadDataFromSpotify(batch); } return result; }, new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 5 }); var saveImageBlock = new ActionBlock(async album => { try { _logger.Debug($"Getting image for {album.id}"); var filename = await _spotifyImageHandler.SaveSmallestAlbumImageAsync(album); _logger.Debug($"Saving image filename for {album.id}"); await _spotifyAlbumManager.UpdateAlbumImageAsync(album.id, filename); } catch (Exception ex) { _logger.Error(ex, $"Could not save album image. Album id: {album.id}"); } }, new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 15 }); var linkOptions = new DataflowLinkOptions { PropagateCompletion = true }; getAlbumInfoBlock.LinkTo(saveImageBlock, linkOptions, dto => dto != null); getAlbumInfoBlock.LinkTo(DataflowBlock.NullTarget(), linkOptions); var albumsToUpdate = await GetAlbumIdsWithMissingImageAsync(); _logger.Debug($"Will fetch image for {albumsToUpdate.Count} albums"); foreach (var batch in albumsToUpdate.Batch(20)) { getAlbumInfoBlock.Post(batch); } getAlbumInfoBlock.Complete(); await saveImageBlock.Completion; } private async Task SetSpotifyTrackPropertiesAsync() { var getAlbumInfoBlock = new TransformManyBlock, FullAlbumItem>(async batch => { Func, Task>> safeLoadDataFromSpotify = async b => { try { return await _spotifyWebApi.GetAllAlbumInfoAsync(b); } catch (SpotifyWebAPIException ex) { return null; } }; var result = await safeLoadDataFromSpotify(batch); if (result == null) { result = await safeLoadDataFromSpotify(batch); } return result; }, new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 5 }); var updateAlbumInfoBlock = new ActionBlock(async album => { try { _logger.Debug($"Saving album data for {album.id}"); await _spotifyAlbumManager.UpdateAlbumPropertiesAsync(album); } catch (Exception ex) { _logger.Error(ex, $"Could not update album properties for album id: {album.id}"); ErrorLoggingManager.Instance.LogError(ex); } }, new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 5 }); var linkOptions = new DataflowLinkOptions { PropagateCompletion = true }; getAlbumInfoBlock.LinkTo(updateAlbumInfoBlock, linkOptions, dto => dto != null); getAlbumInfoBlock.LinkTo(DataflowBlock.NullTarget(), linkOptions); var albumsToUpdate = await GetAlbumIdsWithMissingDataAsync(); _logger.Debug($"Will fetch album data for {albumsToUpdate.Count} albums"); foreach (var batch in albumsToUpdate.Batch(100)) { getAlbumInfoBlock.Post(batch); } getAlbumInfoBlock.Complete(); await updateAlbumInfoBlock.Completion; } public async Task> GetAlbumIdsWithMissingImageAsync() { var albumIds = new List(); using (var conn = await DatabaseHandler.GetOpenReadOnlyConnectionAsync()) { const string sql = "SELECT AlbumId FROM tblSpotifyAlbum WHERE SmallImageFilename IS NULL"; var cmd = new MySqlCommand(sql, conn); var reader = await cmd.ExecuteReaderAsync(); while (await reader.ReadAsync()) { albumIds.Add(reader.GetString(0)); } reader.Close(); } return albumIds; } public async Task> GetAlbumIdsWithMissingDataAsync() { var albumIds = new List(); using (var conn = await DatabaseHandler.GetOpenReadOnlyConnectionAsync()) { const string sql = "SELECT AlbumId FROM tblSpotifyAlbum WHERE Popularity IS NULL OR Copyright IS NULL OR Label IS NULL OR ReleaseDate IS NULL OR ReleaseDatePrecision IS NULL"; var cmd = new MySqlCommand(sql, conn); var reader = await cmd.ExecuteReaderAsync(); while (await reader.ReadAsync()) { albumIds.Add(reader.GetString(0)); } reader.Close(); } return albumIds; } } }