using MoreLinq; using MySql.Data.MySqlClient; using NLog; using Sony.Filtr.Database; using Sony.Filtr.ErrorLogging; using Sony.Filtr.Playlists.Spotify; using Sony.Filtr.SpotifyWebAPI; using Sony.Filtr.Utility.Extensions; using System; using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; using Sony.Filtr.Playlists.Models; using Sony.Filtr.Tasks.Helpers; using System.Threading.Tasks.Dataflow; namespace Sony.Filtr.Tasks.Tasks.Spotify { public class SpotifyTrackAccousticDetailsTask : IScheduledTask { private readonly SpotifyPlaylistManager _spotifyPlaylistManager; private readonly SpotifyWebApi _spotifyWebApi; private readonly UpdatePlaylistsHelper _updatePlaylistsHelper; private readonly Logger _logger; public SpotifyTrackAccousticDetailsTask(SpotifyPlaylistManager spotifyPlaylistManager, SpotifyWebApi spotifyWebApi, UpdatePlaylistsHelper updatePlaylistsHelper) { _spotifyPlaylistManager = spotifyPlaylistManager; _spotifyWebApi = spotifyWebApi; _updatePlaylistsHelper = updatePlaylistsHelper; _logger = LogManager.GetLogger("SetSpotifyTrackAccousticDetails"); } public async Task ExecuteAsync(Guid scheduledTaskLogId) { await SetSpotifyTrackPropertiesAsync(); await SetSpotifyTrackPopularityAsync(); return null; } private async Task SetSpotifyTrackPopularityAsync() { var loadTracksBlock = new TransformBlock, IEnumerable>(async ids => { try { return await this._spotifyWebApi.GetAllTracksAsync(ids); } catch (Exception ex) { _logger.Error(ex, $"Could not load tracks from Spotify"); ErrorLoggingManager.Instance.LogError(ex); } return null; }, new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 1 }); var saveTracksBlock = new ActionBlock>(async tracks => { try { var spotifyTracks = tracks.Select(t => _updatePlaylistsHelper.BuildSpotifyPlaylistTrack(t)).ToList(); await _spotifyPlaylistManager.AddOrUpdateSpotifyTracksAsync(spotifyTracks); } catch (Exception ex) { _logger.Error(ex, $"Could not save tracks popularity from Spotify"); ErrorLoggingManager.Instance.LogError(ex); } }, new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 5 }); loadTracksBlock.LinkTo(saveTracksBlock, new DataflowLinkOptions { PropagateCompletion = true }, dto => dto != null); loadTracksBlock.LinkTo(DataflowBlock.NullTarget>(), new DataflowLinkOptions { PropagateCompletion = true }); try { var tracksToUpdate = await GetTrackIdsWithoutPopularityAsync(); foreach (var batch in tracksToUpdate.Batch(100)) { await loadTracksBlock.SendAsync(batch); } loadTracksBlock.Complete(); await saveTracksBlock.Completion; } catch (Exception ex) { ErrorLoggingManager.Instance.LogError(ex); _logger.Error(ex, "Could not load popularity data for tracks. Exception: {1}", ex); } } private async Task SetSpotifyTrackPropertiesAsync() { try { var tracksToUpdate = await GetTrackIdsWithoutPropertiesAsync(); var batches = tracksToUpdate.Batch(100).ToList(); await batches.ItemIndex().ForEachAsync(5, async batch => { try { _logger.Debug($"Fetching accoustic data for batch {batch.Index} / {batches.Count}"); IEnumerable audioFeatures = await _spotifyWebApi.GetAllAudioFeatureForTracksAsync(batch.Item.ToList()); _logger.Debug($"Saving accoustic data for batch {batch.Index} / {batches.Count}"); await _spotifyPlaylistManager.SaveAudioFeaturesForTracksAsync(audioFeatures); } catch (Exception ex) { _logger.Error(ex); ErrorLoggingManager.Instance.LogError(ex); } }); } catch (Exception ex) { ErrorLoggingManager.Instance.LogError(ex); _logger.Error(ex, "Could not load acoustic details for tracks. Exception: {1}", ex); } } public async Task> GetTrackIdsWithoutPropertiesAsync() { var trackIds = new List(); using (var conn = await DatabaseHandler.GetOpenReadOnlyConnectionAsync()) { var sql = "SELECT TrackId FROM tblSpotifyTrack2 " + "WHERE Danceability IS NULL OR Energy IS NULL OR 'Key' IS NULL OR Loudness IS NULL OR Mode IS NULL OR Speechiness IS NULL " + "OR Acousticness IS NULL OR Instrumentalness IS NULL OR Liveness IS NULL OR Valence IS NULL OR Tempo IS NULL "; var cmd = new MySqlCommand(sql, conn); var reader = await cmd.ExecuteReaderAsync(); while (await reader.ReadAsync()) { trackIds.Add(reader.GetString(0)); } reader.Close(); } return trackIds; } private async Task> GetTrackIdsWithoutPopularityAsync() { var trackIds = new List(); using (var conn = await DatabaseHandler.GetOpenReadOnlyConnectionAsync()) { var sql = "SELECT TrackId FROM tblSpotifyTrack2 WHERE Popularity IS NULL"; var cmd = new MySqlCommand(sql, conn); using(var reader = await cmd.ExecuteReaderAsync()) { while (await reader.ReadAsync()) { trackIds.Add(reader.GetString(0)); } } } return trackIds; } } }