using System; using System.Collections.Concurrent; using System.Collections.Generic; using System.Globalization; using System.IO; using System.Linq; using System.Threading.Tasks; using MoreLinq; using MySql.Data.MySqlClient; using NLog; using Sony.Filtr.Buzz; using Sony.Filtr.Contracts.Definitions; using Sony.Filtr.Contracts.Entities; using Sony.Filtr.Contracts.Entities.Buzz; using Sony.Filtr.Core.EditorialPlaylists; using Sony.Filtr.Core.SpotifyUserFollowers; using Sony.Filtr.Database; using Sony.Filtr.ErrorLogging; using Sony.Filtr.SpotifyImages; using Sony.Filtr.SpotifyWebAPI; using Sony.Filtr.Utility.Extensions; namespace Sony.Filtr.Tasks.Tasks.Spotify { public class ImportSpotifyUserFollowers : IScheduledTask { private readonly SpotifyWebApi _spotifyWebApi; private readonly BuzzAccountManager _buzzAccountManager; private readonly EditorialPlaylistManager _editorialPlaylistManager; private readonly ErrorLoggingManager _errorLoggingManager; private readonly SpotifyUserFollowersManager _spotifyUserFollowersManager; private readonly SpotifyImageDownloader _spotifyImageHandler; private readonly Logger _logger; public ImportSpotifyUserFollowers(SpotifyWebApi spotifyWebApi, BuzzAccountManager buzzAccountManager, EditorialPlaylistManager editorialPlaylistManager, ErrorLoggingManager errorLoggingManager, SpotifyUserFollowersManager spotifyUserFollowersManager, SpotifyImageDownloader spotifyImageHandler) { _spotifyWebApi = spotifyWebApi; _buzzAccountManager = buzzAccountManager; _editorialPlaylistManager = editorialPlaylistManager; _errorLoggingManager = errorLoggingManager; _spotifyUserFollowersManager = spotifyUserFollowersManager; _spotifyImageHandler = spotifyImageHandler; var applicationName = "ImportSpotifyUserFollowers"; _errorLoggingManager.CurrentApplicationName = applicationName; _logger = LogManager.GetLogger(applicationName); } public async Task ExecuteAsync(Guid scheduledTaskLogId) { const string fileDirectory = @"spotifyUserFollowers\"; const string tempFileName = @"spotifyUserFollowersImport.csv"; const string filePath = fileDirectory + tempFileName; if (!Directory.Exists(fileDirectory)) { Directory.CreateDirectory(fileDirectory); } try { ConcurrentDictionary userFollowers = new ConcurrentDictionary(); _logger.Debug("Starting import of spotify user followers..."); var allBuzzUsers = await _buzzAccountManager.GetBuzzUsersAsync(musicServiceId: (int)MusicService.Spotify); var groupedBuzzUsers = allBuzzUsers.GroupBy(b => SpotifyLink.ParseUserLink(b.Username).ExtractSpotifyUserName()).ToDictionary(k => k.Key, v => v.ToList()); var existingFollowerData = await _spotifyUserFollowersManager.GetFollowerDataAsync(DateTime.Today); var usersWithExistingData = existingFollowerData.Select(d => d.Username).ToHashSet(); var spotifyUsers = allBuzzUsers.Select(b => b.Username).Where(u => !usersWithExistingData.Contains(u)).Select(SpotifyLink.ParseUserLink).Distinct().ToList(); _logger.Debug("Importing follower data for " + spotifyUsers.Count + " users..."); await spotifyUsers.ForEachAsync(10, async spotifyUserLink => { var spotifyUserName = spotifyUserLink.ExtractSpotifyUserName(); _logger.Debug("Fetching follower data for {0}", spotifyUserName); try { var userInfo = await _spotifyWebApi.GetUserAsync(spotifyUserName); if (userInfo != null) { userFollowers.TryAdd(spotifyUserName, userInfo.followers.total ?? 0); var buzzUsers = groupedBuzzUsers.GetValueOrDefault(spotifyUserName); if (buzzUsers != null) { var imageFilename = await _spotifyImageHandler.SaveUserImageAsync(userInfo); foreach (var buzzUser in buzzUsers) { buzzUser.DisplayName = userInfo.display_name; buzzUser.Subscribers = userInfo.followers?.total; buzzUser.Image = imageFilename; buzzUser.Error = false; await _buzzAccountManager.UpdateBuzzUserAsync(buzzUser, new List { "Displayname", "Subscribers", "Image", "Error" }); } } } else { _logger.Debug("Could not fetch follower data for {0}", spotifyUserName); await SetBuzzUserErrorAsync(groupedBuzzUsers, spotifyUserName); } } catch (Exception ex) { _logger.Error(ex, "Error when fetching follower for user {0}.", spotifyUserName); _errorLoggingManager.LogError(ex); await SetBuzzUserErrorAsync(groupedBuzzUsers, spotifyUserName); } }); _logger.Debug("Saving follower data to database..."); await SaveFollowerDataToCsvAsync(filePath, userFollowers, DateTime.Today); await BulkAddFollowerDataAsync(filePath); File.Delete(filePath); _logger.Debug("All done!"); } catch (Exception ex) { _logger.Error(ex, "Exception :( "); _errorLoggingManager.LogError(ex); } return null; } private async Task SetBuzzUserErrorAsync(Dictionary> groupedBuzzUsers, string spotifyUserName) { var buzzUsers = groupedBuzzUsers.GetValueOrDefault(spotifyUserName); if (buzzUsers != null) { foreach(var buzzUser in buzzUsers) { buzzUser.Error = true; await _buzzAccountManager.UpdateBuzzUserAsync(buzzUser, new List { "Error" }); } } } private async Task SaveFollowerDataToCsvAsync(string filePath, ConcurrentDictionary userFollowers, DateTime importDate) { CultureInfo cultureInfo = CultureInfo.GetCultureInfo("sv-SE"); using (var saveFile = File.Open(filePath, FileMode.OpenOrCreate)) { using (var fileWriter = new StreamWriter(saveFile)) { foreach (var user in userFollowers) { var csvLine = string.Join("|", user.Key, importDate.ToString(cultureInfo.DateTimeFormat.ShortDatePattern), user.Value); await fileWriter.WriteLineAsync(csvLine); } } } } private async Task BulkAddFollowerDataAsync(string filePath) { using (var conn = await DatabaseHandler.GetOpenConnectionAsync()) { var bulkLoader = new MySqlBulkLoader(conn); bulkLoader.Local = true; bulkLoader.FileName = filePath; bulkLoader.TableName = "tblSpotifyUserFollowerLog"; bulkLoader.FieldTerminator = "|"; bulkLoader.LineTerminator = Environment.NewLine; bulkLoader.ConflictOption = MySqlBulkLoaderConflictOption.Replace; await bulkLoader.LoadAsync(); } } } }