using System; using System.Collections.Concurrent; using System.Collections.Generic; using System.IO; using System.Linq; using System.Text; using System.Threading.Tasks; using MySql.Data.MySqlClient; using NLog; using Sony.Filtr.Contracts.Entities; using Sony.Filtr.Database; using Sony.Filtr.Playlists.Spotify; using Sony.Filtr.SpotifyWebAPI; using Sony.Filtr.Utility.Extensions; namespace Sony.Filtr.Worker.Onetime { public class PlaylistsInSpotifyAnalytics { private readonly SpotifyPlaylistManager _spotifyPlaylistManager; private readonly SpotifyWebApi _spotifyWebApi; private readonly Logger _logger; public PlaylistsInSpotifyAnalytics(SpotifyPlaylistManager spotifyPlaylistManager, SpotifyWebApi spotifyWebApi) { _spotifyPlaylistManager = spotifyPlaylistManager; _spotifyWebApi = spotifyWebApi; _logger = LogManager.GetLogger("PlaylistsInSpotifyAnalytics"); } public async Task ExecuteAsync() { var playlists = await GetTempUntrackedPlaylistsStreamsInAnalyticsAsync(); ConcurrentBag tempPlaylists = new ConcurrentBag(); await playlists.ForEachAsync(10, async playlistUri => { try { Console.WriteLine($"Fetching {playlistUri.Key}"); var playlistLink = new SpotifyLink(playlistUri.Key, urlDecode:true); var playlistId = playlistLink.ExtractPlaylistID(); var spotifyPlaylist = await _spotifyWebApi.GetPlaylistByIdAsync(playlistId, "name,description,followers.total"); var tempPlaylist = new TempPlaylist() { PlaylistUri = playlistUri.Key, Name = spotifyPlaylist.name, Description = spotifyPlaylist.description, Followers = spotifyPlaylist.followers?.total ?? 0, Streams = playlistUri.Value, }; tempPlaylists.Add(tempPlaylist); await SetUntrackedPlaylistDataAsync(tempPlaylist); Console.WriteLine($"Done with {playlistUri.Key}"); } catch (Exception e) { _logger.Error(e); } }); FileInfo file = new FileInfo($"PlaylistsInSpotifyAnalytics.csv"); using (var fileStream = file.OpenWrite()) { fileStream.Write(Encoding.UTF8.GetPreamble(), 0, Encoding.UTF8.GetPreamble().Length); using (var fileWriter = new StreamWriter(fileStream)) { fileWriter.WriteLine(string.Join(";", "PlaylistUri", "Name", "Streams", "Followers")); foreach (var playlistTrackData in tempPlaylists) { var row = string.Join(";", playlistTrackData.PlaylistUri, playlistTrackData.Name, playlistTrackData.Streams, playlistTrackData.Followers); await fileWriter.WriteLineAsync(row); } } } //var allTrackedPlaylistUris = _spotifyPlaylistManager.GetAllPlaylistUris(); //var allTrackedPlaylistUrisDic = allTrackedPlaylistUris.ToHashSet(); //var date = new DateTime(2017, 11, 4); ////var playlistsWithStreams = await GetPlaylistsStreamsInAnalyticsAsync(date); ////var untrackedPlaylists2 = playlistsWithStreams.Where(p => !allTrackedPlaylistUris.Contains(p.Key)); ////foreach (var untrackedPlaylist in untrackedPlaylists2) ////{ //// _logger.Info($"Untracked Playlist: {untrackedPlaylist.Key} had {untrackedPlaylist.Value} streams"); ////} //ConcurrentDictionary allPlaylistStreams = new ConcurrentDictionary(); //var analyticsplaylists = await GetPlaylistsInAnalyticsAsync(date); //var untrackedPlaylists = analyticsplaylists.AsParallel().Where(p => !allTrackedPlaylistUrisDic.Contains(p)).ToList(); //int playlistIndex = 0; //var totalUntrackedPlaylists = untrackedPlaylists.Count; //await untrackedPlaylists.ForEachAsync(10, async playlist => //{ // var playlistStreams = await GetPlaylistsStreamsInAnalyticsAsync(date, playlist); // allPlaylistStreams.TryAdd(playlist, playlistStreams); // Interlocked.Increment(ref playlistIndex); // _logger.Debug($"{playlistIndex} {totalUntrackedPlaylists}"); //}); //foreach (var untrackedPlaylist in allPlaylistStreams) //{ // _logger.Info($"Untracked Playlist: {untrackedPlaylist} had {untrackedPlaylist.Value} streams"); //} ////var untrackedPlaylists = playlists.Except(allTrackedPlaylistUris); } private async Task SetUntrackedPlaylistDataAsync(TempPlaylist playlist) { using (var conn = await DatabaseHandler.GetOpenConnectionAsync()) { const string sql = "UPDATE _TempUntrackedPlaylists2 SET Name = @name, Description = @description, Followers = @followers WHERE PlaylistUri = @playlistUri"; using (var cmd = new MySqlCommand(sql, conn)) { cmd.Parameters.AddWithValue("@name", playlist.Name); cmd.Parameters.AddWithValue("@description", playlist.Description); cmd.Parameters.AddWithValue("@followers", playlist.Followers); cmd.Parameters.AddWithValue("@playlistUri", playlist.PlaylistUri); await cmd.ExecuteNonQueryAsync(); } } } private async Task> GetTempUntrackedPlaylistsStreamsInAnalyticsAsync() { Dictionary playlists = new Dictionary(); using (var conn = await DatabaseHandler.GetOpenReadOnlyConnectionAsync()) { const string sql = "SELECT PlaylistUri, Streams56Days FROM _TempUntrackedPlaylists2 " + "WHERE Name IS NULL " + "ORDER BY Streams56Days DESC"; using (var cmd = new MySqlCommand(sql, conn)) { using (var reader = await cmd.ExecuteReaderAsync()) { while (await reader.ReadAsync()) { var playlistUri = reader.GetString(0); var streams = reader.GetInt32(1); playlists.Add(playlistUri, streams); } } } } return playlists; } } public class TempPlaylist { public string PlaylistUri { get; set; } public string Name { get; set; } public string Description { get; set; } public long Streams { get; set; } public long Followers { get; set; } } }