using System; using System.Collections.Concurrent; using System.Collections.Generic; using System.IO; using System.Linq; using System.Text; using System.Threading.Tasks; using MySql.Data.MySqlClient; using NLog; using Sony.Filtr.ApolloAPI; using Sony.Filtr.ApolloAPI.Models; using Sony.Filtr.Applications; using Sony.Filtr.Core.Buzz; using Sony.Filtr.Database; using Sony.Filtr.DistributedCaching; using Sony.Filtr.PlaylistGeneration.MusicInfoStore; using Sony.Filtr.Utility; using Sony.Filtr.Utility.Extensions; namespace Sony.Filtr.Tasks.Tasks.Spotify { public class FetchPopularArtists : IScheduledTask { private readonly LastFmManager _lastFmManager; private readonly ApplicationInstanceManager _applicationInstanceManager; private readonly BuzzArtistManager _buzzArtistManager; private readonly Logger _logger; private readonly IApolloWebApi _vendorApi; private const string FileDirectory = @"popularartists\"; private const string TempFileName = @"artistImport.csv"; private const string FilePath = FileDirectory + TempFileName; private const int GlobalAmount = 500; private const int CountryAmount = 500; public FetchPopularArtists(LastFmManager lastFmManager, IApolloWebApi vendorApi, ApplicationInstanceManager applicationInstanceManager, BuzzArtistManager buzzArtistManager) { _lastFmManager = lastFmManager; _vendorApi = vendorApi; _applicationInstanceManager = applicationInstanceManager; _buzzArtistManager = buzzArtistManager; _logger = LogManager.GetLogger("FetchPopularArtists"); } public async Task ExecuteAsync(Guid scheduledTaskLogId) { if (!Directory.Exists(FileDirectory)) { Directory.CreateDirectory(FileDirectory); } try { var allArtistNames = await GetArtists(); var existingBuzzArtists = _buzzArtistManager.GetBuzzArtists(); allArtistNames = allArtistNames.Except(existingBuzzArtists.Select(a => a.ArtistName), StringComparer.InvariantCultureIgnoreCase).ToList(); _logger.Info("Total amount of unique artist names: " + allArtistNames.Count + "\n"); _logger.Info("Getting Spotify info for artists..."); var spotifyArtistInfo = await GetArtistSpotifyInfoAsync(allArtistNames); _logger.Info("Adding " + spotifyArtistInfo.Count + " Spotify artists to database..."); await SaveTracksToCsvAsync(spotifyArtistInfo); await BulkAddArtistDataAsync(); File.Delete(FilePath); _logger.Info("Done adding to database..."); } catch (Exception ex) { _logger.Error(ex); } return null; } private async Task> GetArtists() { List allArtists = new List(); _logger.Info("Getting top " + GlobalAmount + " global artists..."); var globalArtists = (await GetPopularArtistsFromLastFmAsync(GlobalAmount)).ToList(); allArtists.AddRange(globalArtists.ToList()); _logger.Info("Getting top " + CountryAmount + " artists per market..."); var countryArtists = (await GetPopularArtistsFromLastFmForAllMarketsAsync(CountryAmount)).ToList(); allArtists.AddRange(countryArtists); allArtists = allArtists.Distinct().ToList(); _logger.Info("Total amount of artists: " + allArtists.Count + "\n"); return allArtists; } private async Task> GetPopularArtistsFromLastFmForAllMarketsAsync(int amount) { var marketCountries = _applicationInstanceManager.GetApplications().Where(a=> a.GACountryName != null ).Select(a => a.GACountryName); marketCountries = marketCountries.Except(new List() { "All", "Global", "-", "se", "workout", "workout all", "CA", "Latin"}, StringComparer.InvariantCultureIgnoreCase); var countryTopArtists = new ConcurrentBag(); await marketCountries.ForEachAsync(5, async marketCountry => { _logger.Info("Getting top artists for market {0}...", marketCountry); try { var topArtists = await _lastFmManager.GetGeoTopArtistAsync(marketCountry, amount); topArtists.ForEach(a => countryTopArtists.Add(a)); } catch (Exception ex) { _logger.Error(ex, "Exception in GetPopularArtistsFromLastFMForAllMarketsAsync for market {0}. Exception:{1}", marketCountry, ex); } }); return countryTopArtists.ToList(); } private async Task> GetPopularArtistsFromLastFmAsync(int amount, string market = null) { return await _lastFmManager.GetChartTopArtistsAsync(amount); } private async Task> GetArtistSpotifyInfoAsync(IEnumerable topArtists) { var spotifyArtists = new ConcurrentBag(); await topArtists.ForEachAsync(5, async artist => { try { _logger.Debug("Doing lookup on Spotify for artist {0}...", artist); //const string fields = "artists.items(name,id)"; var searchResponse = await _vendorApi.SearchArtistsAsync(artist); if (searchResponse.artists != null && searchResponse.artists.items.Any()) { var artistMatch = searchResponse.artists.items.FirstOrDefault(a => TextUtility.LooksLikeSpecifiedArtist(a.name, artist)); if (artistMatch != null) { spotifyArtists.Add(artistMatch); _logger.Debug("Found artist {0} on Spotify...", artist); } else { _logger.Warn("Could not find artist {0} on Spotify...", artist); } } } catch (Exception ex) { _logger.Error(ex, "Error searching for artist {0}.", artist); } }); return spotifyArtists.ToList(); } private async Task SaveTracksToCsvAsync(IEnumerable artistInformation) { using (var saveFile = File.Open(FilePath, FileMode.OpenOrCreate)) { using (var fileWriter = new StreamWriter(saveFile, Encoding.UTF8)) { foreach (var artist in artistInformation) { var csvLine = string.Join("|", string.Empty, artist.name, artist.id, string.Empty, 0, // Set followers to 0, will be updated through scheduled task 0 // Set playlist followers to 0, will be updated once a profile has been set ); await fileWriter.WriteLineAsync(csvLine); } } } } private async Task BulkAddArtistDataAsync() { using (var conn = await DatabaseHandler.GetOpenConnectionAsync()) { var bulkLoader = new MySqlBulkLoader(conn) { Local = true, FileName = FilePath, TableName = "tblBuzzArtist", FieldTerminator = "|", LineTerminator = Environment.NewLine, ConflictOption = MySqlBulkLoaderConflictOption.Ignore, CharacterSet = "utf8" }; await bulkLoader.LoadAsync(); } } } }