using System; using System.Collections.Concurrent; using System.Collections.Generic; using System.Data.Common; using System.Linq; using System.Threading.Tasks; using MoreLinq; using MySql.Data.MySqlClient; using NLog; using Sony.Filtr.AdminSearch; using Sony.Filtr.AdminSearch.Models; using Sony.Filtr.AppleMusic; using Sony.Filtr.AppleMusic.Data; using Sony.Filtr.AppleMusic.Data.Internal; using Sony.Filtr.AppleMusic.Data.Streams; using Sony.Filtr.AppleMusic.Playlists; using Sony.Filtr.Buzz; using Sony.Filtr.Contracts.Definitions; using Sony.Filtr.Contracts.Entities.Buzz; using Sony.Filtr.Database; using Sony.Filtr.DspWebAPI; using Sony.Filtr.ErrorLogging; using Sony.Filtr.Utility.Extensions; namespace Sony.Filtr.Tasks.Tasks.Spotify { public class AdminSearchIndexTask : IScheduledTask { private readonly AdminSearchManager _adminSearchManager; private readonly AppleMusicPlaylistManager _appleMusicPlaylistManager; private readonly AppleMusicStreamsManager _appleMusicStreamsManager; private readonly BuzzAccountManager _buzzAccountManager; private readonly IDspWebApi _dspWebApi; private readonly Logger _logger; private string[] markets = new[] { "us" }; public AdminSearchIndexTask(AdminSearchManager adminSearchManager, AppleMusicPlaylistManager appleMusicPlaylistManager, AppleMusicStreamsManager appleMusicStreamsManager, BuzzAccountManager buzzAccountManager, IDspWebApi dspWebApi) { _adminSearchManager = adminSearchManager; _appleMusicPlaylistManager = appleMusicPlaylistManager; _appleMusicStreamsManager = appleMusicStreamsManager; _buzzAccountManager = buzzAccountManager; _dspWebApi = dspWebApi; _logger = LogManager.GetLogger("AdminSearchIndexTaskLogger"); } public async Task ExecuteAsync(Guid scheduledTaskLogId) { try { await _adminSearchManager.SetupIndexAndMappingAsync(); _logger.Debug("Begin IndexPlaylistsAsync"); await IndexSpotifyPlaylistsAsync(); await IndexAppleMusicPlaylistsAsync(); await IndexAmazonPlaylistsAsync(); _logger.Debug("Done with IndexPlaylistsAsync"); } catch (Exception ex) { ErrorLoggingManager.Instance.LogError(ex); _logger.Error(ex, "Indexing playlists task failed :( {0}", ex); } return null; } private async Task IndexAppleMusicPlaylistsAsync() { var playlists = await _appleMusicPlaylistManager.GetPlaylistsAsync(); var allPlaylists = playlists.Items; ConcurrentBag allGlobalPlaylistStreams = new ConcurrentBag(); var batches = allPlaylists.Select(p => p.Id).ToList().Batch(100).ToList(); await batches.ItemIndex().ForEachAsync(1, async playlistIdBatch => { _logger.Debug($"Getting streams for batch {playlistIdBatch.Index} of {batches.Count}"); try { var playlistStreams = _appleMusicStreamsManager.GetAppleMusicContainerStreamsSummaries(playlistIdBatch.Item.ToList(), "global"); playlistStreams.ForEach(p => allGlobalPlaylistStreams.Add(p)); } catch (Exception e) { Console.WriteLine(e); } }); var playlistStreamsDic = allGlobalPlaylistStreams.ToDictionary(k => k.ContainerId, v => v); var buzzUsers = await _buzzAccountManager.GetBuzzUsersAsync(MusicService.AppleMusic); var buzzUserDic = buzzUsers.ToDictionary(k => k.Username, v => v); _logger.Debug($"Found {allPlaylists.Count} playlists to index."); var playlistIndexItems = allPlaylists.Select(p => BuildAppleMusicPlaylistIndexItem(p, buzzUserDic, playlistStreamsDic)); var addBatches = playlistIndexItems.Batch(1000).ToList(); await addBatches.ItemIndex().ForEachAsync(1, async batchItem => { try { _logger.Debug($"Begin indexing batch {batchItem.Index} of {addBatches.Count}."); var playlistsInBatch = batchItem.Item.ToList(); await _adminSearchManager.IndexAppleMusicPlaylistsAsync(playlistsInBatch); _logger.Debug($"Done indexing batch {batchItem.Index} of {addBatches.Count}."); } catch (Exception ex) { _logger.Error(ex, $"Error for {batchItem.Index} of {addBatches.Count}. Continuing anyway."); ErrorLoggingManager.Instance.LogError(ex); } }); var playlistsToRemove = await GetAllAppleMusicPlaylistsToRemoveAsync(); var removeBatches = playlistsToRemove.Batch(1000).ToList(); await removeBatches.ItemIndex().ForEachAsync(1, async batchItem => { try { _logger.Debug($"Begin remove indexing batch {batchItem.Index} of {removeBatches.Count}."); var playlistsInBatch = batchItem.Item.ToList(); await _adminSearchManager.RemoveAppleMusicPlaylistsFromIndexAsync(playlistsInBatch); _logger.Debug($"Done remove indexing batch {batchItem.Index} of {removeBatches.Count}."); } catch (Exception ex) { _logger.Error(ex, $"Error for remove batch {batchItem.Index} of {removeBatches.Count}. Continuing anyway."); ErrorLoggingManager.Instance.LogError(ex); } }); } private async Task IndexAmazonPlaylistsAsync() { _logger.Debug($"Begin Indexing AmazonPlaylists."); List allPlaylistsWithNecessaryMarket = new List(); int count = 0, offset = 0; int limit = 5000; do { try { var playlists = await _dspWebApi.GetAmazonPlaylists(limit, offset); if (playlists != null) { count = playlists.Count(); offset = offset + count; allPlaylistsWithNecessaryMarket.AddRange(playlists.Where(x=> markets.Contains(x.CountryCode))); } } catch (Exception ex) { _logger.Error(ex, $"Error in fetch AmazonPlaylists. Limit: {limit}, offset: {offset}"); ErrorLoggingManager.Instance.LogError(ex); } } while (count > 0); var removeBatches = allPlaylistsWithNecessaryMarket.Batch(1000).ToList(); await removeBatches.ItemIndex().ForEachAsync(1, async batchItem => { try { _logger.Debug($"Begin remove indexing batch {batchItem.Index} of {removeBatches.Count}."); var playlistsInBatch = batchItem.Item.ToList(); await _adminSearchManager.RemoveAmazonPlaylistsFromIndexAsync(playlistsInBatch); _logger.Debug($"Done remove indexing batch {batchItem.Index} of {removeBatches.Count}."); } catch (Exception ex) { _logger.Error(ex, $"Error for remove batch {batchItem.Index} of {removeBatches.Count}. Continuing anyway."); ErrorLoggingManager.Instance.LogError(ex); } }); var addBatches = allPlaylistsWithNecessaryMarket.Batch(1000).ToList(); await addBatches.ItemIndex().ForEachAsync(1, async batchItem => { try { _logger.Debug($"Begin indexing batch {batchItem.Index} of {removeBatches.Count}."); var playlistsInBatch = batchItem.Item.ToList(); await _adminSearchManager.IndexAmazonPlaylistsAsync(playlistsInBatch); _logger.Debug($"Done indexing batch {batchItem.Index} of {removeBatches.Count}."); } catch (Exception ex) { _logger.Error(ex, $"Error for adding batch {batchItem.Index} of {removeBatches.Count}. Continuing anyway."); ErrorLoggingManager.Instance.LogError(ex); } }); _logger.Debug($"Done Indexing AmazonPlaylists."); } private static AppleMusicPlaylistIndexItem BuildAppleMusicPlaylistIndexItem(AppleMusicPlaylist playlist, Dictionary buzzUserDic, Dictionary playlistStreamsDic) { var buzzUser = playlist.CuratorId.HasValue ? buzzUserDic.GetValueOrDefault(playlist.CuratorId.ToString()) : null; var streams = playlistStreamsDic.GetValueOrDefault(playlist.Id); return new AppleMusicPlaylistIndexItem() { PlaylistId = playlist.Id, Name = playlist.Name, ArtworkUrl = playlist.Artwork, BuzzCategoryId = buzzUser?.BuzzCategoryId, Curator = buzzUser?.DisplayName, UpdateDate = playlist.LatestUpdate, CountryCode = buzzUser?.CountryCode ?? "us", Streams56Days = streams?.Streams56Days ?? 0, }; } private async Task IndexSpotifyPlaylistsAsync() { var playlists = await GetAllSpotifyPlaylistItemsAsync(); _logger.Debug($"Found {playlists.Count} playlists to index."); var addBatches = playlists.Batch(1000).ToList(); await addBatches.ItemIndex().ForEachAsync(1, async batchItem => { try { _logger.Debug($"Begin indexing batch {batchItem.Index} of {addBatches.Count}."); var playlistsInBatch = batchItem.Item.ToList(); await _adminSearchManager.IndexSpotifyPlaylistsAsync(playlistsInBatch); _logger.Debug($"Done indexing batch {batchItem.Index} of {addBatches.Count}."); } catch (Exception ex) { _logger.Error(ex, $"Error for {batchItem.Index} of {addBatches.Count}. Continuing anyway."); ErrorLoggingManager.Instance.LogError(ex); } }); var playlistsToRemove = await GetAllSpotifyPlaylistsToRemoveAsync(); var removeBatches = playlistsToRemove.Batch(1000).ToList(); await removeBatches.ItemIndex().ForEachAsync(1, async batchItem => { try { _logger.Debug($"Begin remove indexing batch {batchItem.Index} of {removeBatches.Count}."); var playlistsInBatch = batchItem.Item.ToList(); await _adminSearchManager.RemoveSpotifyPlaylistsFromIndexAsync(playlistsInBatch); _logger.Debug($"Done remove indexing batch {batchItem.Index} of {removeBatches.Count}."); } catch (Exception ex) { _logger.Error(ex, $"Error for remove batch {batchItem.Index} of {removeBatches.Count}. Continuing anyway."); ErrorLoggingManager.Instance.LogError(ex); } }); } private async Task> GetAllSpotifyPlaylistsToRemoveAsync() { var playlists = new List(); using (var conn = await DatabaseHandler.GetOpenReadOnlyConnectionAsync()) { const string sql = @" SELECT p.PlaylistUri FROM tblSpotifyPlaylist AS p LEFT JOIN tblPlaylistIgnoredPlaylist AS ip ON ip.playlistId = p.playlistUri AND ip.musicServiceId = @spotifyMusicServiceId WHERE p.Removed = 1 OR ip.playlistId IS NOT NULL;"; using (var cmd = new MySqlCommand(sql, conn)) { cmd.Parameters.AddWithValue("@spotifyMusicServiceId", MusicService.Spotify); using (var reader = await cmd.ExecuteReaderAsync()) { while (await reader.ReadAsync()) { var playlistUri = reader.GetString("PlaylistUri"); playlists.Add(new SpotifyPlaylistIndexItem() { PlaylistUri = playlistUri }); } } } } return playlists; } private async Task> GetAllAppleMusicPlaylistsToRemoveAsync() { var removedPlaylists = new List(); using (var conn = await DatabaseHandler.GetOpenReadOnlyConnectionAsync()) { const string sql = "SELECT PlaylistId " + "FROM tblAppleMusicPlaylist AS p " + "LEFT JOIN tblPlaylistIgnoredPlaylist AS ip ON ip.playlistId = p.id AND ip.musicServiceId = @appleMusicMusicServiceId " + "WHERE p.Removed = 1 OR ip.playlistId IS NOT NULL "; using (var cmd = new MySqlCommand(sql, conn)) { cmd.Parameters.AddWithValue("@appleMusicMusicServiceId", MusicService.AppleMusic); using (var reader = await cmd.ExecuteReaderAsync()) { while (await reader.ReadAsync()) { var playlistId = reader.GetString("PlaylistId"); removedPlaylists.Add(new AppleMusicPlaylistIndexItem() { PlaylistId = playlistId }); } } } } return removedPlaylists; } private async Task> GetAllSpotifyPlaylistItemsAsync() { var playlists = new List(); using (var conn = await DatabaseHandler.GetOpenReadOnlyConnectionAsync()) { const string sql = "SELECT p.PlaylistUri, p.Name, p.Description, p.Image, p.User, p.CountryCode, p.Duration, " + "p.TrackCount, p.UpdateDate, p.BuzzCategoryId, p.TrackLatestAdded, p.Public, " + "f.Followers, u.DisplayName " + "FROM tblSpotifyPlaylist AS p " + "LEFT JOIN tblSpotifyPlaylistFollowers AS f ON f.PlaylistId = p.PlaylistId " + "LEFT JOIN BuzzUser AS u ON u.Username = p.User AND u.ServiceType = 0 " + "LEFT JOIN tblPlaylistIgnoredPlaylist AS ip ON ip.playlistId=p.playlistUri AND ip.musicServiceId=@spotifyMusicServiceId " + "WHERE p.Removed = 0 AND ip.playlistId IS NULL " + "GROUP BY p.PlaylistUri"; using (var cmd = new MySqlCommand(sql, conn)) { cmd.Parameters.AddWithValue("@spotifyMusicServiceId", MusicService.Spotify); using (var reader = await cmd.ExecuteReaderAsync()) { while (await reader.ReadAsync()) { var playlist = BuildPlaylistIndexItem(reader); playlists.Add(playlist); } } } } return playlists; } private static SpotifyPlaylistIndexItem BuildPlaylistIndexItem(DbDataReader reader) { var playlistUri = reader.GetString("PlaylistUri"); var playlistName = reader.GetString("Name"); var description = reader.GetString("description"); var image = reader.GetString("image"); var user = reader.GetString("user"); var countryCode = reader.GetString("countryCode"); var duration = reader.GetIntOrDefault("duration") ?? 0; var trackCount = reader.GetIntOrDefault("trackCount") ?? 0; var updatedate = reader.GetDateTime("updateDate"); var buzzCategoryId = reader.GetIntOrDefault("buzzCategoryId"); var @public = reader.GetBoolean("public"); var followers = reader.GetIntOrDefault("followers") ?? 0; var trackLatestAdded = reader.GetUtcDateTimeOrDefault("TrackLatestAdded"); var userDisplayName = reader.GetString("DisplayName"); var playlist = new SpotifyPlaylistIndexItem() { PlaylistUri = playlistUri, Name = playlistName, Description = description, Image = image, User = user, CountryCode = countryCode, Duration = duration, TrackCount = trackCount, Followers = followers, BuzzCategoryId = buzzCategoryId, UpdateDate = updatedate, TrackLatestAdded = trackLatestAdded, Public = @public, UserDisplayName = userDisplayName }; return playlist; } } }