using MoreLinq;
using NLog;
using Sony.Filtr.Buzz;
using Sony.Filtr.Contracts.Abstractions;
using Sony.Filtr.Contracts.Definitions;
using Sony.Filtr.Contracts.Entities;
using Sony.Filtr.Contracts.Entities.Buzz;
using Sony.Filtr.Core.Buzz;
using Sony.Filtr.Core.EditorialPlaylists;
using Sony.Filtr.Core.SpotifyUserCountries;
using Sony.Filtr.Core.TrackPopularity;
using Sony.Filtr.ErrorLogging;
using Sony.Filtr.Playlists;
using Sony.Filtr.Playlists.Models;
using Sony.Filtr.Playlists.Spotify;
using Sony.Filtr.Playlists.Spotify.Model;
using Sony.Filtr.SpotifyImages;
using Sony.Filtr.SpotifyWebAPI;
using Sony.Filtr.SpotifyWebAPI.Model;
using Sony.Filtr.Tasks.Helpers;
using Sony.Filtr.Utility.Extensions;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using System.Diagnostics;
using Sony.Filtr.Tasks.EventSource;
using Sony.Filtr.ApolloAPI;
using static Sony.Filtr.Playlists.Spotify.SpotifyPlaylistManager;
using Sony.Filtr.Utility;
using System.Threading.Tasks.Dataflow;
using System.Collections.ObjectModel;
using System.Threading;
using System.Text;
using Sony.Filtr.Functional;
using Sony.Filtr.Core.SpotifyWeekly;
//using Sony.Filtr.Functional;

namespace Sony.Filtr.Tasks.Tasks
{
    public class UpdatePlaylistInfoTask : UpdatePlaylistInfoBase
    {
        static class PlaylistsFilter
        {
            public const string FilePathWithPlaylistIdsToUse = @"output\UpdatePlaylistInfoTask_PlaylistIdsToUse.txt";

            private static Logger _logger;

            static PlaylistsFilter()
            {
                _logger = LogManager.GetLogger("UpdatePlaylistInfo_PlaylistFilter");
            }

            public static bool ShouldUsePredefinedPlaylists()
            {
                bool shouldFilter = File.Exists(FilePathWithPlaylistIdsToUse);

                _logger.Warn($"Should filter: {shouldFilter}");

                return shouldFilter;
            }

            public static string[] GetPlaylistIdsToUse()
            {
                if (!File.Exists(FilePathWithPlaylistIdsToUse)) { return Array.Empty<string>(); }

                return File.ReadAllLines(FilePathWithPlaylistIdsToUse);
            }
        }

        private readonly SpotifyWebApi _spotifyWebApi;
        protected readonly IApolloWebApi _apolloWebApi;
        private readonly EditorialPlaylistManager _editorialPlaylistManager;

        private readonly IServiceAccountManager _serviceAccountManager;
        private readonly SpotifyTrackPopularityManager _spotifyTrackPopularityManager;
        private readonly SpotifyImageDownloader _spotifyImageHandler;
        private readonly BuzzManager _buzzManager;

        private readonly IgnoredPlaylistsManager _ignoredPlaylistsManager;
        private readonly SpotifyWeeklyManager _spotifyWeeklyManager;
        private readonly AsyncLogger _loggerFull;
        private readonly AsyncLogger _loggerCurrentProcessed;
        private readonly AsyncLogger _loggerPlaylistLoad;
        private readonly AsyncLogger _loggerErrors;
        private readonly AsyncLogger _copyTracklistHistoryDuration;
        private readonly AsyncLogger _playlistDatesDuration;
        private readonly AsyncLogger _playlistEditorialsDuration;
        private readonly AsyncLogger _playlistRelatedDataDuration;
        private readonly AsyncLogger _trackPopularitiesAddedDuration;
        private readonly AsyncLogger _fullUpdateDuration;
        private readonly AsyncLogger _loggerGetLatestPopularitiesDuration;
        private readonly AsyncLogger _loggerTracklistChanges;
        private readonly AsyncLogger _loggerPersonalized;

        private readonly ActionTracker blockCountTracker;

        private Func<string, Task<Result<PlaylistResponse>>> getAuthenticatedSpotifyPlaylistByIdAsync;
        private Func<string, int, int, Task<Result<PlaylistTracksResponse>>> getAuthenticatedPlaylistTracksByIdAsync;

        public UpdatePlaylistInfoTask(SpotifyWebApi spotifyWebApi,
            IApolloWebApi apolloWebApi,
            EditorialPlaylistManager editorialPlaylistManager,
            SpotifyPlaylistManager spotifyPlaylistManager,
            SpotifyUserCountryManager spotifyUserCountryManager,
            BuzzAccountManager buzzAccountManager,
            IServiceAccountManager serviceAccountManager,
            SpotifyTrackPopularityManager spotifyTrackPopularityManager,
            SpotifyImageDownloader spotifyImageHandler,
            BuzzManager buzzManager,
            SpotifyPlaylistHistoricTrackListManager historicTrackListManager,
            IgnoredPlaylistsManager ignoredPlaylistsManager,
            UpdatePlaylistsHelper updatePlaylistsHelper,
            SpotifyWeeklyManager spotifyWeeklyManager)
            : base(spotifyPlaylistManager, historicTrackListManager, updatePlaylistsHelper, buzzAccountManager, spotifyUserCountryManager)
        {
            _spotifyWebApi = spotifyWebApi;
            _apolloWebApi = apolloWebApi;
            _editorialPlaylistManager = editorialPlaylistManager;
            _serviceAccountManager = serviceAccountManager;
            _spotifyTrackPopularityManager = spotifyTrackPopularityManager;
            _spotifyImageHandler = spotifyImageHandler;
            _buzzManager = buzzManager;
            _ignoredPlaylistsManager = ignoredPlaylistsManager;
            _spotifyWeeklyManager = spotifyWeeklyManager;

            _loggerCurrentProcessed = AsyncLogger.GetLogger("UpdatePlaylistInfo_Processed");

            _loggerFull = AsyncLogger.GetLogger("UpdatePlaylistInfo");
            _loggerPlaylistLoad = AsyncLogger.GetLogger("Playlist_Load");

            _copyTracklistHistoryDuration = AsyncLogger.GetLogger("CopyTracklistHistoryDuration");
            _playlistDatesDuration = AsyncLogger.GetLogger("PlaylistDatesDuration");
            _playlistEditorialsDuration = AsyncLogger.GetLogger("PlaylistEditorialsDuration");
            _playlistRelatedDataDuration = AsyncLogger.GetLogger("PlaylistRelatedDataDuration");
            _trackPopularitiesAddedDuration = AsyncLogger.GetLogger("TrackPopularitiesAddDuration");
            _fullUpdateDuration = AsyncLogger.GetLogger("PlaylistFullUpdateDuration");
            _loggerGetLatestPopularitiesDuration = AsyncLogger.GetLogger("PLaylistLatestTrackPopularitiesLoadDuration");
            _loggerTracklistChanges = AsyncLogger.GetLogger("TracklistChanges");
            _loggerErrors = AsyncLogger.GetLogger("Errors");
            _loggerPersonalized = AsyncLogger.GetLogger("PersonalizedDuration");

            this.blockCountTracker = new ActionTracker(Path.Combine("logs", $"blocksCount_{DateTime.Now.ToString("yyyy_MM_dd")}.txt"), 10);

            ShouldCheckForPersonalized = Maybe.GetAppSettingsBooleanOrDefault("UpdatePlaylistInfo_Should_Check_For_Personalized_Playlists", true);
            SavePlaylistImageToAmazon = Maybe.GetAppSettingsBooleanOrDefault("UpdatePlaylistInfo_Save_Playlist_Image_To_Amazon", true);
        }

        protected override async Task<ScheduledTaskLog> ExecuteJobAsync(Guid scheduledTaskLogId)
        {
            if (!Directory.Exists("output"))
            {
                Directory.CreateDirectory("output");
            }

            var cts = new CancellationTokenSource();
            var killProcessTask = StartTheadToTrackForceKillTime(cts.Token);

            var taskLog = await UpdatePlaylistsAsync();

            await killProcessTask;

            return taskLog;
        }

        private Task StartTheadToTrackForceKillTime(CancellationToken cancellationToken)
        {
            DateTime killTime = GetDateTimeWhenToKillACurrentJob();

            _loggerCurrentProcessed.InfoAsync(() => $"Kill time: {killTime}").FireAndForget();

            return TaskHelper.CreateLongRunningThread(
                cancellationToken, 
                () =>
                {
                    var now = DateTime.Now;
                    if (now > killTime)
                    {
                        _loggerCurrentProcessed.FatalAsync(() => $"Process kill time reached ({killTime}). Aborting process...").FireAndForget();

                        Process.GetCurrentProcess().Kill();
                    }
                },
                TimeSpan.FromSeconds(60));
        }

        protected virtual bool DoFinalCleanup => Maybe.GetAppSettingsBooleanOrDefault("UpdatePlaylistInfo_DoFinalCleanup", true);

        protected virtual bool ForcePlaylistsForNotifications => Maybe.GetAppSettingsBooleanOrDefault("UpdatePlaylistInfo_ForcePlaylistsForNotifications", false);

        private async Task<ScheduledTaskLog> UpdatePlaylistsAsync()
        {
            ClearCacheForBuzzImportUsers();

            _loggerFull.WarnAsync(() => "Begin UpdatePlaylistNew.").FireAndForget();

            if (this.ForcePlaylistsForNotifications)
            {
                await ForceTopPlaylistsForNotificationsAsync();
            }

            ApolloUpdatePlaylistInfoEventSource.Log.PrepareDataStart();
            var input = new SpotifyPlaylistInputData()
            {
                Playlists = await GetPlaylistToProcessAsync(),
                PersonalizedPlaylistsServiceAccount = _serviceAccountManager.GetServiceAccount(MusicService.Spotify, "filtr"),
                PersonalizedPlaylists = await GetPersonalizedPlaylistIdsAsync()
            };
            ApolloUpdatePlaylistInfoEventSource.Log.PrepareDataStop();
            _loggerPlaylistLoad.WarnAsync(() => $"Done loading playlists from DB. ({input.Playlists.Count} playlists, {input.Playlists.SelectMany(p => p.EditorialPlaylists).Count()} total editorial playlists)").FireAndForget();
            _loggerCurrentProcessed.WarnAsync(() => $"Done loading playlists from DB. ({input.Playlists.Count} playlists, {input.Playlists.SelectMany(p => p.EditorialPlaylists).Count()} total editorial playlists)").FireAndForget();


            var authenticatedApi = CreateAuthenticatedSpotifyApi(input.PersonalizedPlaylistsServiceAccount);
            this.getAuthenticatedSpotifyPlaylistByIdAsync = CreateAuthenticatedSpotifyGetPlaylistByIdFunction(authenticatedApi);
            this.getAuthenticatedPlaylistTracksByIdAsync = CreateAuthenticatedSpotifyGetPlaylistTracksByIdAsyncFunction(authenticatedApi);

            await GetAndUpdateSpotifyPlaylistDataAsync(input);

            if (this.DoFinalCleanup)
            {
                ((Action)_spotifyTrackPopularityManager.ClearOldPopularityLogValues)
                    .TryCatch()
                    .OnFailure(result => this.LogError(result.Exception, () => "Could not clear old popularity"))();

                ClearCacheForBuzzImportUsers();

                var finalStatePlaylists = await GetPlaylistToProcessAsync();

                _loggerCurrentProcessed.WarnAsync(() => $"Playlists left to process {finalStatePlaylists.Count()}").FireAndForget();
            }

            _loggerFull.WarnAsync(() => "Done UpdatePlaylist").FireAndForget();

            return null;
        }

        private async Task ForceTopPlaylistsForNotificationsAsync()
        {
            var apolloPlaylistsTask = this._apolloWebApi.GetPlaylistFavorites();
            var weeklyPlaylsitsTask = this._spotifyWeeklyManager.GetLatestTopPlaylistIds();

            await Task.WhenAll(apolloPlaylistsTask, weeklyPlaylsitsTask);

            await this._spotifyPlaylistManager.SetSavetracklistForPlaylistsAsync(
                apolloPlaylistsTask.Result.SpotifyPlaylistIds.Union(weeklyPlaylsitsTask.Result).Distinct(),
                true);
        }

        private AuthenticatedSpotifyWebApi CreateAuthenticatedSpotifyApi(ServiceAccount personalizedPlaylistsServiceAccount)
        {
            string clientId = GetPersonalizedSpotifyClientId();
            string secretId = GetPersonalizedSpotifyClientSecret();

            return (String.IsNullOrWhiteSpace(clientId) || String.IsNullOrWhiteSpace(secretId)) ?
              _spotifyWebApi.GetAuthenticatedSession(personalizedPlaylistsServiceAccount, _serviceAccountManager) :
              SpotifyWebApi.GetAuthenticatedSession(personalizedPlaylistsServiceAccount, _serviceAccountManager, clientId, secretId);
        }

        private Func<string, int, int, Task<Result<PlaylistTracksResponse>>> CreateAuthenticatedSpotifyGetPlaylistTracksByIdAsyncFunction(AuthenticatedSpotifyWebApi authApi)
        {
            var f = ((Func<string, string, int?, int?, string, Task<PlaylistTracksResponse>>)authApi.GetPlaylistTracksByIdAsync)
                .Tuple()
                .Timeout(TimeSpan.FromSeconds(10))
                .Retry(2, TimeSpan.FromSeconds(3))
                .TryCatch()
                .OnFailure((tuple, result) => this.LogError(result.Exception, () => $"Authenticated GetPlaylistTracksByIdAsync Failed. PlaylistId: '{tuple.Item1}'"));

            return async (playlistId, limit, offset) => {
                return await f((playlistId, null, limit, offset, null));
            };
        }

        private Func<string, Task<Result<PlaylistResponse>>> CreateAuthenticatedSpotifyGetPlaylistByIdFunction(AuthenticatedSpotifyWebApi authApi)
        {
            var f = ((Func<string, string, string, Task<PlaylistResponse>>)authApi.GetPlaylistByIdAsync)
            .Tuple()
            .Timeout(TimeSpan.FromSeconds(10))
            .Retry(2, TimeSpan.FromSeconds(3))
            .TryCatch()
            .OnFailure((tuple, result) => this.LogError(result.Exception, () => $"Authenticated GetPlaylistByIdAsync Failed. PlaylistId: '{tuple.Item1}'"));

            return async playlistId => {

                return await f((playlistId, null, null));
            };
        }

        private async Task<HashSet<string>> GetPersonalizedPlaylistIdsAsync()
        {
            return (await _spotifyPlaylistManager.GetAllPersonalizedPlaylistIdsAsync()).ToHashSet();
        }

        private int GetNumberOfDaysToSubstractFromTodayToUseAsPivotForTracksLastAddedFilter()
        {
            return Maybe.GetAppSettingsIntOrDefault("UpdatePlaylistInfo_TracksLastAdded_DaysToSubstract", 0);
        }

        private bool ShouldUsePlaylistsWithTrackAddedAfterPivot()
        {
            return Maybe.GetAppSettingsBooleanOrDefault("UpdatePlaylistInfo_ShouldUse_TracksLastAdded_After_Pivot", false);
        }

        private static Lazy<bool> loadEditoralPlaylists = new Lazy<bool>(() => Maybe.GetAppSettingsBooleanOrDefault("UpdatePlaylistInfo_Should_Load_Editorials", true));
        private static bool ShouldLoadEditorialPlaylists()
        {
            return loadEditoralPlaylists.Value;
        }

        internal virtual async Task<List<SpotifyPlaylistInput>> GetPlaylistToProcessAsync()
        {
            //return new string[] {
            //            "5H765bjiGWNovpZbfPeM3o"
            //"37i9dQZF1DWU19R364DeUc",
            //"37i9dQZF1DWUJF24WXSSyO"
            //"0EAL8l5TB3PxV1aI5Mt5W8",
            //"0ICN1H5Jo25SXOeyljqK8U",
            //"0X7BHuJowHEuiXPH2ujmbV",
            //"0kNpUrIXHcPFtIKNNOq6PX",
            //"1Fv5ClYGqCM0q2S8xm1UsK",
            //"1fA0PSoLiiGphRK2Veuxsm",
            //"2Ir7xB9ogQnuXYtcC7Ntws",
            //"2dYJTWOOYHMJPP6TTaWYOC",
            //"2oo75mKwtyxkj4UW092Wmh",
            //"3EiwSG7AEh1wjGahbuK7Y9",
            //"3YbYzyav8GFvAmwWmsZe71",
            //"3cN18nMp0Eh0cdjrCf8g5f",
            //"4XYT6B0WdLT9e5qM6W4AdN",
            //"4kLB7MjL8op6dRGDKGkiL5",
            //"4sktuaCYk0BRDHSNjiptj7",
            //"4tBk9VMrMqWBjeJktCwlDm",
            //"6eMOnjdbL7XEXSbMxWi9dz",
            //"6sGBZoeZhs7iM9qcnujMdM",
            //"7G49Us4VDu8cNKSrGxnHSq"
            //          }
            //.ItemIndex()
            //    .Select(p => new SpotifyPlaylistInput(
            //        new PlaylistMarker(p.Item, "MjksOTg5Y2M3YzE0ZDVmYTVhMjA5ZmI5MThjYTdlZDFlMmM4OTdmZGZjMw==", "se", p.Index),
            //        new ReadOnlyCollection<EditorialPlaylist>(new EditorialPlaylist[0]),
            //        true))
            //    .ToList();

            var editorialPlaylists = ShouldLoadEditorialPlaylists() ? await _editorialPlaylistManager.GetLightweightEditorialPlaylistsAsync(onlyActive: false) : new List<EditorialPlaylist>();
            editorialPlaylists = await FilterIgnoredPlaylistsAsync(editorialPlaylists);

            bool shouldUsePredefinedPlaylists = PlaylistsFilter.ShouldUsePredefinedPlaylists();
            _loggerPlaylistLoad.InfoAsync(() => $"Loading playlists to process... Should use predefined playlists: {shouldUsePredefinedPlaylists}").FireAndForget();

            List<PlaylistUpdateReference> notUpdatedPlaylists = shouldUsePredefinedPlaylists ?
                await _spotifyPlaylistManager.GetPlaylistsToUpdateAsync(PlaylistsFilter.GetPlaylistIdsToUse()) :
                await _spotifyPlaylistManager.GetNotUpdatedPlaylistsAsync(
                    DateTime.UtcNow,
                    DateTime.UtcNow.AddDays(-this.GetNumberOfDaysToSubstractFromTodayToUseAsPivotForTracksLastAddedFilter()),
                    this.ShouldUsePlaylistsWithTrackAddedAfterPivot());

            _loggerPlaylistLoad.InfoAsync(() => $"Days to substract: {this.GetNumberOfDaysToSubstractFromTodayToUseAsPivotForTracksLastAddedFilter()}. After pivot: {this.ShouldUsePlaylistsWithTrackAddedAfterPivot()}. Total count: {notUpdatedPlaylists.Count}").FireAndForget();

            var updatedPlaylists = await _spotifyPlaylistManager.GetUpdatedPlaylistsAsync(DateTime.UtcNow);

            var updatedPlaylistLookup = updatedPlaylists.Select(p => p.PlaylistId).ToHashSet();
            var notUpdatedEditorialPlaylists = editorialPlaylists.Where(p => !updatedPlaylistLookup.Contains(p.SpotifyLink.ExtractPlaylistID())).ToList();

            var spotifyPlaylistToUpdate = new List<SpotifyPlaylistManager.PlaylistUpdateReference>();
            spotifyPlaylistToUpdate.AddRange(notUpdatedEditorialPlaylists.Select(p => new SpotifyPlaylistManager.PlaylistUpdateReference()
            {
                PlaylistId = p.SpotifyLink.ExtractPlaylistID(),
                SaveTracklist = true,
                CountryCode = p.CountryCode
            }));
            spotifyPlaylistToUpdate.AddRange(notUpdatedPlaylists);
            spotifyPlaylistToUpdate = spotifyPlaylistToUpdate
                .GroupBy(p => p.PlaylistId)
                .Select(p => new SpotifyPlaylistManager.PlaylistUpdateReference()
                {
                    PlaylistId = p.Key,
                    SaveTracklist = p.Any(pi => pi.SaveTracklist),
                    SnapshotId = p.FirstOrDefault(pi => !String.IsNullOrWhiteSpace(pi.SnapshotId))?.SnapshotId,
                    CountryCode = p.FirstOrDefault(pi => !String.IsNullOrWhiteSpace(pi.CountryCode))?.CountryCode
                }).ToList();

            var editorialPlaylistDic = notUpdatedEditorialPlaylists.GroupBy(p => p.SpotifyLink.ExtractPlaylistID()).ToDictionary(k => k.Key, v => v);

            return spotifyPlaylistToUpdate
                .ItemIndex()
                .Select(p => new SpotifyPlaylistInput(
                    new PlaylistMarker(p.Item.PlaylistId, p.Item.SnapshotId, p.Item.CountryCode, p.Index),
                    editorialPlaylistDic.GetValueOrDefault(p.Item.PlaylistId)?.ToList().AsReadOnly() ?? new ReadOnlyCollection<EditorialPlaylist>(new EditorialPlaylist[0]),
                    p.Item.SaveTracklist))
                .ToList();
        }

        private async Task<List<EditorialPlaylist>> FilterIgnoredPlaylistsAsync(List<EditorialPlaylist> playlists)
        {
            var ignoredPlaylists = await _ignoredPlaylistsManager.GetIgnoredPlaylistsAsync();
            var ignoredSpotifyPlaylists = ignoredPlaylists.Where(p => p.MusicServiceId == (int)MusicService.Spotify).Select(p => p.PlaylistId).Distinct().ToHashSet();
            return playlists.Where(p => !ignoredSpotifyPlaylists.Contains(p.SpotifyLinkUri)).ToList();
        }

        private async Task<bool> DetermineAndUpdatePersonalizedSpotifyDataAsync(
            Contracts.Entities.SpotifyPlaylist playlist,
            List<UpdatePlaylistTrack> currentTracks,
            SpotifyPlaylistInputData input)
        {
            bool isCurrentlyPersonalized = false;

            Stopwatch sw = new Stopwatch();
            sw.Start();
            try
            {
                if (_updatePlaylistsHelper.ShouldCheckForPersonalizedPlaylist(playlist))
                {
                    var playlistId = playlist.PlaylistId;

                    if (input.PersonalizedPlaylistsServiceAccount == null)
                    {
                        this.LogError(() => $"Could not find service account for personalization check of playlist { playlistId }");
                    }
                    else
                    {
                        bool trackListChanged = await this.HasTracklistChanged(playlistId, currentTracks);

                        var wasPersonalizedBeforeChange = input.PersonalizedPlaylists.Contains(playlistId);
                        isCurrentlyPersonalized = trackListChanged; //If the tracklist is different between api user and a "regular" user we assume the playlist is personalized

                        this._loggerFull.InfoAsync(() => $"DetermineAndUpdatePersonalizedSpotifyData: playlistId: {playlistId}. wasPersonalizedBeforeChange: {wasPersonalizedBeforeChange}. isCurrentlyPersonalized: {isCurrentlyPersonalized}").FireAndForget();

                        await _spotifyPlaylistManager.UpdatePlaylistPersonalizedStatusAsync(playlistId, isCurrentlyPersonalized, wasPersonalizedBeforeChange);
                    }
                }
            }
            catch (Exception e)
            {
                this.LogError(e, () => $"Error while checking personalization for playlist { playlist?.PlaylistId }");
            }

            sw.Stop();
            //_loggerPersonalized.WarnAsync(() => $"'{playlist.PlaylistId}' DetermineAndUpdatePersonalizedSpotifyDataAsync Duration: {sw.Elapsed.TotalSeconds}secs").FireAndForget();

            return isCurrentlyPersonalized;
        }

        private void ShowFirstTracklistDifference(string playlistId, IEnumerable<UpdatePlaylistTrack> x, IEnumerable<UpdatePlaylistTrack> y)
        {
            Nullable<int> firstNonMatch = x.FirstNonMatchIndex(y, UpdatePlaylistsHelper.GetUpdatePlaylistTrackEqualFunc());

            if (firstNonMatch.HasValue)
            {
                var xElement = x.ElementAt(firstNonMatch.Value);
                var yElement = y.ElementAt(firstNonMatch.Value);

                _loggerFull.InfoAsync(() => $"Playlist '{playlistId}' Tracks at position {firstNonMatch.Value} are different. x.Id: {xElement.Track.Id}. y.Id: {yElement.Track.Id}. x.Name: {xElement.Track.Name}. y.Name: {yElement.Track.Name} x.Added: {xElement.Added}. y.Added: {yElement.Added}");
            }
        }

        private static string GetPersonalizedSpotifyClientId()
        {
            return Maybe.GetAppSettingsStringOrDefault("UpdatePlaylistInfo_Personalized_Spotify_ClientId", String.Empty);
        }

        private static string GetPersonalizedSpotifyClientSecret()
        {
            return Maybe.GetAppSettingsStringOrDefault("UpdatePlaylistInfo_Personalized_Spotify_ClientSecret", String.Empty);
        }

        private async Task<bool> HasTracklistChanged(string playlistId, List<UpdatePlaylistTrack> currentTracks)
        {
            Result<PlaylistResponse> fetchedPlaylistResult = await this.getAuthenticatedSpotifyPlaylistByIdAsync(playlistId);

            if (fetchedPlaylistResult.IsFailed) { return false; }

            PlaylistResponse fetchedPlaylist = fetchedPlaylistResult.Value;

            if (fetchedPlaylist == null) { return false; }

            if (fetchedPlaylist.Tracks.items.Any(t => t.track == null)) { return false; }

            if (currentTracks.StartsWith(
                this.ToUpdatePlaylistTracks(fetchedPlaylist.Tracks.items),
                UpdatePlaylistsHelper.GetUpdatePlaylistTrackEqualFunc()) == false)
            {
                this.ShowFirstTracklistDifference(playlistId, currentTracks, this.ToUpdatePlaylistTracks(fetchedPlaylist.Tracks.items));

                return true;
            }

            int batchSize = 100;
            var batches = fetchedPlaylist.Tracks.total.Batch(batchSize);
            for (int i = 1; i < batches.Length; ++i)
            {
                var trackBatchResult = await this.getAuthenticatedPlaylistTracksByIdAsync(playlistId, batches[i].Limit, batches[i].Offset);
                if(trackBatchResult.IsFailed) { return false; }

                PlaylistTracksResponse trackBatch = trackBatchResult.Value;
                if (trackBatch == null)
                {
                    return false;
                }

                if (trackBatch.items.Any(t => t.track == null)) { return false; }

                if (currentTracks
                        .Skip(batchSize * i)
                        .StartsWith(
                            this.ToUpdatePlaylistTracks(trackBatch.items),
                            UpdatePlaylistsHelper.GetUpdatePlaylistTrackEqualFunc()) == false)
                {
                    this.ShowFirstTracklistDifference(playlistId, currentTracks.Skip(batchSize * i), this.ToUpdatePlaylistTracks(trackBatch.items));

                    return true;
                }
            }

            return false;
        }

        private IEnumerable<UpdatePlaylistTrack> ToUpdatePlaylistTracks(IEnumerable<PlaylistTrackItem> items)
        {
            return items
                    .Where(p => p.track?.id != null)
                    .Index()
                    .Select(x => _updatePlaylistsHelper.BuildSpotifyPlaylistTrack(x.Value, x.Key))
                    .ToList();
        }

        private async Task SavePlaylistChangesAsync(Contracts.Entities.SpotifyPlaylist spotifyPlaylist, Contracts.Entities.SpotifyPlaylist dbPlaylist)
        {
            if (spotifyPlaylist.SnapshotId != dbPlaylist.SnapshotId)
            {
                var changes = new List<PlaylistChangeHistoryEntry>();

                changes.Add(new PlaylistChangeHistoryEntry()
                {
                    PlaylistId = spotifyPlaylist.PlaylistId,
                    Date = DateTime.Today,
                    ChangeType = PlaylistChangeType.SnapshotId,
                    OldValue = dbPlaylist.SnapshotId,
                    NewValue = spotifyPlaylist.SnapshotId,
                    Timestamp = DateTime.Now
                });

                if (spotifyPlaylist.Name != dbPlaylist.Name)
                {
                    changes.Add(new PlaylistChangeHistoryEntry()
                    {
                        PlaylistId = spotifyPlaylist.PlaylistId,
                        Date = DateTime.Today,
                        ChangeType = PlaylistChangeType.Name,
                        NewValue = spotifyPlaylist.Name,
                        OldValue = dbPlaylist.Name,
                        Timestamp = DateTime.Now
                    });
                }

                await _spotifyPlaylistManager.SavePlaylistChangeAsync(changes);
            }
        }

        private static int SpotifyFetchConcurrentThreadsCount
        {
            get { return Maybe.GetAppSettingsIntOrDefault("UpdatePlaylistInfo_Spotify_Fetch_Concurrent_Threads_Count", 5); }
        }

        private static int UpdateDateConcurrentThreadsCount
        {
            get { return Maybe.GetAppSettingsIntOrDefault("UpdatePlaylistInfo_Update_Date_Concurrent_Threads_Count", 2); }
        }

        private static int FullUpdateConcurrentThreadsCount
        {
            get { return Maybe.GetAppSettingsIntOrDefault("UpdatePlaylistInfo_Full_Update_Concurrent_Threads_Count", 2); }
        }

        private static int RelatedDataUpdateConcurrentThreadsCount
        {
            get { return Maybe.GetAppSettingsIntOrDefault("UpdatePlaylistInfo_Related_Data_Update_Concurrent_Threads_Count", 5); }
        }

        private static int GetPlaylistLatestTrackPopularitiesConcurrentThreadsCount
        {
            get { return Maybe.GetAppSettingsIntOrDefault("UpdatePlaylistInfo_Get_Playlist_Latest_Track_Popularities_Concurrent_Threads_Count", 2); }
        }

        private static int CopyTracklistHistoryConcurrentThreadsCount
        {
            get { return Maybe.GetAppSettingsIntOrDefault("UpdatePlaylistInfo_Copy_Tracklist_History_Concurrent_Threads_Count", 2); }
        }

        private static int AddTrackPopularitiesConcurrentThreadsCount
        {
            get { return Maybe.GetAppSettingsIntOrDefault("UpdatePlaylistInfo_Add_Track_Popularities_Concurrent_Threads_Count", 2); }
        }

        private static DateTime GetDateTimeWhenToKillACurrentJob()
        {
            DateTime killTime = DateTime.ParseExact(Maybe.GetAppSettingsStringOrDefault("UpdatePlaylistInfo_KillTime_HH:mm:ss", "23:58:00"), "HH:mm:ss", null);

            if (DateTime.Now > killTime)
            {
                killTime = killTime.AddDays(1);
            }

            return killTime;
        }

        private static int GetVendorApiTimeout()
        {
            return Maybe.GetAppSettingsIntOrDefault("UpdatePlaylistInfo_VendorApi_Timeout", 8);
        }

        private readonly bool ShouldCheckForPersonalized;

        private readonly bool SavePlaylistImageToAmazon;

        private async Task GetAndUpdateSpotifyPlaylistDataAsync(SpotifyPlaylistInputData input)
        {
            int totalPlaylistsToProcess = input.Playlists.Count;
            var progressTracker = new ProgressTracker(totalPlaylistsToProcess);
            progressTracker.NextAsync(0).FireAndForget();

            string progressChartFilePath = $"logs/ProgressChart_{DateTime.Now.ToString("yyyy_MM_dd")}.bmp";

            #region spotifyPlaylistLoaderBlock
            Func<string, string, Task<SpotifySpecificData>> getSpotifySpecificData = this.GetSpotifyPlaylistSpecificDataAsync;

            TransformBlock<SpotifyPlaylistInput, Result<SpotifyPlaylistToSave>> spotifyPlaylistLoaderBlock =
                getSpotifySpecificData
                .Tuple()
                .Timeout(TimeSpan.FromSeconds(GetVendorApiTimeout()))
                .Retry(1)
                .Map<SpotifyPlaylistInput, SpotifyPlaylistToSave, (string, string), SpotifySpecificData>(
                    inputTransform: async i => (i.PlaylistMarker.PlaylistId, i.PlaylistMarker.SnapshotId),
                    outputTransform: async (tuple, spotifyData, i) =>
                    {
                        if (!spotifyData.PlaylistRemoved)
                        {
                            FixPlaylistBuzzCategoryAndCountry(spotifyData, i.PlaylistMarker.CountryCode);
                        }

                        return new SpotifyPlaylistToSave(i, spotifyData, this.ShouldSaveTracklistHistory(spotifyData.BuzzCategoryId, i.PlaylistMarker.PlaylistId));
                    }
                )
                .TryCatch()
                .OnFailure((i, toSaveResult) => this.LogError(toSaveResult.Exception, () => $"GetSpotifyPlaylistSpecificDataAsync failed. PlaylistId: '{i.PlaylistMarker.PlaylistId}'"))
                .OnSuccess((i, toSaveResult) =>
                {
                    i.PlaylistMarker.StageComplete(Stages.SpotifyLoad);
                    _loggerFull.InfoAsync(() => $"2222222222 Loaded: {toSaveResult.Value.SpotifyPlaylistData?.PlaylistId} Tracks: {toSaveResult.Value.SpotifyPlaylistData?.Tracks?.Count()}, BuzzCategory: {toSaveResult.Value.SpotifyPlaylistData?.Username}");
                })
                .AsTransformBlock(new ExecutionDataflowBlockOptions { EnsureOrdered = false, MaxDegreeOfParallelism = SpotifyFetchConcurrentThreadsCount, SingleProducerConstrained = false, BoundedCapacity = 400 });
            #endregion spotifyPlaylistLoaderBlock

            #region getPlaylistLatestTrackPopularities
            Func<string, Task<IEnumerable<SpotifyTrackPopularityLogValue>>> getPlaylistLatestTrackPopularities = _spotifyTrackPopularityManager.GetPlaylistLatestTrackPopularitiesAsync;

            TransformManyBlock<Result<SpotifyPlaylistToSave>, Result<SpotifyTrackPopularityLogValue>> getPlaylistLatestTrackPopularitiesBlock =
                getPlaylistLatestTrackPopularities
                .Timeout(TimeSpan.FromSeconds(10))
                .Retry(3)
                .Duration((id, logs, duration) => _loggerGetLatestPopularitiesDuration.TraceAsync(() => $"Loaded {logs.Count()} latest track popularities for playlist '{id}' in {duration.TotalSeconds} secs"))
                .Map<SpotifyPlaylistToSave, IEnumerable<SpotifyTrackPopularityLogValue>, string, IEnumerable<SpotifyTrackPopularityLogValue>>(
                    inputTransform: toSave => toSave.Input.PlaylistMarker.PlaylistId,
                    outputTransform: (playlistId, logs, toSave) => {
                        var date = DateTime.Now.Date;
                        foreach (var log in logs)
                        {
                            log.Day = date;
                        }
                        
                        return logs;
                    }
                )
                .TryCatch()
                .OnFailure((toSave, result) => this.LogError(result.Exception, () => $"Could not read playlist track popularities. PlaylistId '{toSave.SpotifyPlaylistData.PlaylistId}'"))
                .SelectMany()
                .AcceptResult()
                .AsTransformManyBlock(new ExecutionDataflowBlockOptions() { EnsureOrdered = false, MaxDegreeOfParallelism = GetPlaylistLatestTrackPopularitiesConcurrentThreadsCount });
            #endregion getPlaylistLatestTrackPopularities

            BroadcastBlock<Result<SpotifyPlaylistToSave>> playlistToSaveBroadcastBlock = new BroadcastBlock<Result<SpotifyPlaylistToSave>>(i => i);

            #region copyTracklistHistoryBlock
            Func<string, Task> copySpotifyTracklistHistoryForToday = _historicTrackListManager.CopySpotifyTrackListHistoryForTodayAsync;

            TransformBlock<Result<SpotifyPlaylistToSave>, Result<SpotifyPlaylistToSave>> copyTracklistHistoryBlock =
                copySpotifyTracklistHistoryForToday
                .ToUnit()
                .Timeout(TimeSpan.FromSeconds(10))
                .Retry(3)
                .Duration((id, unit, duration) => _copyTracklistHistoryDuration.TraceAsync(() => $"Tracklist copied for '{id}' in {duration.TotalSeconds}sec"))
                .Map<SpotifyPlaylistToSave, SpotifyPlaylistToSave, string, Unit>(
                    inputTransform: toSave => toSave.SpotifyPlaylistData.PlaylistId,
                    outputTransform: (playlistId, unit, toSave) => toSave
                )
                .TryCatch()
                .OnFailure((toSave, result) => this.LogError(result.Exception, () => $"Could not copy playlist tracklist history {toSave.SpotifyPlaylistData.PlaylistId}"))
                .OnSuccess((toSave, result) => _loggerFull.InfoAsync(() => $"444444444444 Copied tracklist history: {toSave.IndexToString()}"))
                .AcceptResult()
                .AsTransformBlock(new ExecutionDataflowBlockOptions() { EnsureOrdered = false, MaxDegreeOfParallelism = CopyTracklistHistoryConcurrentThreadsCount });

            var updateDatesBatchBlock = new BatchBlock<Result<SpotifyPlaylistToSave>>(100);
            #endregion copyTracklistHistoryBlock

            #region updatePlaylistsDatesBlock
            Func<IEnumerable<string>, Task> updatePlaylistsUpdateDatesAsync = this._spotifyPlaylistManager.UpdatePlaylistsDateAsync;

            ActionBlock<IEnumerable<Result<SpotifyPlaylistToSave>>> updatePlaylistsDatesBlock =
                updatePlaylistsUpdateDatesAsync
                .ToUnit()
                .Timeout(TimeSpan.FromSeconds(10))
                .Retry(2)
                .Duration((ids, unit, duration) => _playlistDatesDuration.TraceAsync(() => $"Playlists dates updated in {duration.TotalSeconds}sec"))
                .Map<IEnumerable<Result<SpotifyPlaylistToSave>>, Unit, IEnumerable<string>, Unit>(
                    inputTransform: toSaveResults => toSaveResults.Select(r => r.Value.SpotifyPlaylistData.PlaylistId).ToList(),
                    outputTransform: (ids, unit, toSaveResults) => Unit.Default
                )
                .TryCatch()
                .OnFailure((toSaveResults, unitResult) => this.LogError(unitResult.Exception, () => $"Could NOT update playlists dates (tblSpotifyPlaylist UpdateDate/tblSpotifyPlaylistTrackList2 Timestamp)"))
                .OnSuccess((toSaveResults, unitResult) =>
                {
                    var builder = new StringBuilder();
                    foreach (var playlist in toSaveResults.Select(r => r.Value))
                    {
                        playlist.Input.PlaylistMarker.StageComplete(Stages.SavePlaylist);
                        builder.AppendLine($"9999999999 Playlist update date updated '{playlist.IndexToString()}'");
                    }

                    _loggerFull.InfoAsync(builder.ToString);
                    _loggerCurrentProcessed.InfoAsync(builder.ToString);
                })
                .OnSuccess((toSaveResults, unitResult) => { progressTracker.NextAsync(toSaveResults.Count()).FireAndForget(); })
                .AsActionBlock(new ExecutionDataflowBlockOptions() { EnsureOrdered = false, MaxDegreeOfParallelism = UpdateDateConcurrentThreadsCount });
            #endregion updatePlaylistsDatesBlock

            #region updateEditorialPlaylistsBlock
            Func<(IEnumerable<EditorialPlaylist> EditorialPlaylists, SpotifySpecificData Playlist), Task> updateEditorialPlaylistsAsync = UpdateEditorialPlaylistsAsync;

            TransformBlock<Result<SpotifyPlaylistToSave>, Result<SpotifyPlaylistToSaveWithTracks>> updateEditorialPlaylistsBlock =
                updateEditorialPlaylistsAsync
                .ToUnit()
                .Timeout(TimeSpan.FromSeconds(10))
                .Retry(1)
                .Duration((tuple, unit, duration) => _playlistEditorialsDuration.TraceAsync(() => $"Playlist '{tuple.Playlist.PlaylistId}' editorials updated in {duration.TotalSeconds}sec"))
                .Map<SpotifyPlaylistToSave, SpotifyPlaylistToSaveWithTracks, (IEnumerable<EditorialPlaylist> EditorialPlaylists, SpotifySpecificData Playlist), Unit>
                (
                    inputTransform: toSave => (toSave.Input.EditorialPlaylists, toSave.SpotifyPlaylistData),
                    outputTransform: (tuple, unit, toSave) =>
                    {
                        try
                        {
                            List<UpdatePlaylistTrack> tracks = toSave.SpotifyPlaylistData.Tracks
                               .ItemIndex()
                               .Select(t => _updatePlaylistsHelper.BuildSpotifyPlaylistTrack(t.Item, t.Index))
                               .ToList();

                            SpotifyPlaylist spotifyPlaylist = toSave.SpotifyPlaylistData.ToDbEntity(tracks, _updatePlaylistsHelper.CalculateTrackLatestAdded(tracks));

                            return new SpotifyPlaylistToSaveWithTracks(toSave, spotifyPlaylist, tracks);
                        }
                        catch (Exception ex)
                        {
                            this.LogError(ex, () => $"Could not output transform editorial block. PlaylistId '{toSave.SpotifyPlaylistData?.PlaylistId}'");
                            return new SpotifyPlaylistToSaveWithTracks(toSave, null, null);
                        }
                    }
                )
                .TryCatch()
                .OnFailure((toSave, result) => this.LogError(result.Exception, () => $"Could not update editorial playlist {toSave.SpotifyPlaylistData.PlaylistId}"))
                .OnSuccess((toSave, result) => _loggerFull.InfoAsync(() => $"444444444 Editorials updated: {toSave.IndexToString()}"))
                .AcceptResult()
                .AsTransformBlock(new ExecutionDataflowBlockOptions() { EnsureOrdered = false, MaxDegreeOfParallelism = 1 });
            #endregion updateEditorialPlaylistsBlock

            #region playlistSaveRelatedDataAndAddTracklistHistoryBlock
            Func<SpotifyPlaylistInputData, (SpotifyPlaylistToSave, List<UpdatePlaylistTrack>, SpotifyPlaylist), Task> saveRelatedDataAsync = SaveRelatedDataAsync;

            TransformBlock<Result<SpotifyPlaylistToSave>, Result<SpotifyPlaylistToSave>> playlistSaveRelatedDataAndAddTracklistHistoryBlock =
                saveRelatedDataAsync
                .Partial(input)
                .ToUnit()
                .Timeout(TimeSpan.FromMinutes(2))
                .Retry(2)
                .Duration((tuple, unit, duration) => _playlistRelatedDataDuration.TraceAsync(() => $"Playlist '{tuple.Item1.Input.PlaylistMarker.PlaylistId}' related data updated in {duration.TotalSeconds}sec"))
                .Map<SpotifyPlaylistToSave, SpotifyPlaylistToSave, (SpotifyPlaylistToSave, List<UpdatePlaylistTrack>, SpotifyPlaylist), Unit>(
                    inputTransform: toSave =>
                    {
                        List<UpdatePlaylistTrack> tracks = toSave.SpotifyPlaylistData.Tracks
                                        .ItemIndex()
                                        .Select(t => _updatePlaylistsHelper.BuildSpotifyPlaylistTrack(t.Item, t.Index))
                                        .ToList();

                        SpotifyPlaylist spotifyPlaylist = toSave.SpotifyPlaylistData.ToDbEntity(tracks, _updatePlaylistsHelper.CalculateTrackLatestAdded(tracks));

                        return (toSave, tracks, spotifyPlaylist);
                    },
                    outputTransform: (tuple, unit, toSave) => toSave
                )
                .TryCatch()
                .OnFailure((toSave, result) => this.LogError(result.Exception, () => $"Could not save related data for '{toSave.SpotifyPlaylistData.PlaylistId}'"))
                .OnSuccess((toSave, result) =>
                {
                    toSave.Input.PlaylistMarker.StageComplete(Stages.SaveRelated);
                    _loggerFull.InfoAsync(() => $"5555555555 Related data updated: {toSave.IndexToString()}");
                })
                .AcceptResult()
                .AsTransformBlock(new ExecutionDataflowBlockOptions() { EnsureOrdered = false, MaxDegreeOfParallelism = RelatedDataUpdateConcurrentThreadsCount });
            #endregion playlistSaveRelatedDataAndAddTracklistHistoryBlock

            #region playlistToTrackPopularitiesBlock
            var playlistToTrackPopularitiesBlock = new TransformManyBlock<Result<SpotifyPlaylistToSaveWithTracks>, Result<SpotifyTrackPopularityLogValue>>(
                toSave => toSave.Value.Tracks.Select(t => Result<SpotifyTrackPopularityLogValue>.Success(new SpotifyTrackPopularityLogValue() { TrackLink = new SpotifyLink(t.Track.Id), Popularity = t.Track.Popularity, Day = DateTime.Today.Date }))
            , new ExecutionDataflowBlockOptions() { MaxDegreeOfParallelism = 1 });
            #endregion playlistToTrackPopularitiesBlock

            BatchBlock<Result<SpotifyTrackPopularityLogValue>> trackPopularitiesBatchBlock = new BatchBlock<Result<SpotifyTrackPopularityLogValue>>(20000);

            #region addPlaylistTrackPopularitiesBlock
            Func<IEnumerable<SpotifyTrackPopularityLogValue>, Task> addPopularityLogValuesAsync = _spotifyTrackPopularityManager.AddPopularityLogValuesAsync;

            ActionBlock<IEnumerable<Result<SpotifyTrackPopularityLogValue>>> addPlaylistTrackPopularitiesBlock =
                addPopularityLogValuesAsync
                .ToUnit()
                .Timeout(TimeSpan.FromSeconds(20))
                .Retry(2)
                .Map<IEnumerable<Result<SpotifyTrackPopularityLogValue>>, Unit, IEnumerable<SpotifyTrackPopularityLogValue>, Unit>(
                    inputTransform: results => results.Select(r => r.Value).ToArray(),
                    outputTransform: (logs, unit, logs2) => Unit.Default
                )
                .Duration((logs, unit, duration) => _trackPopularitiesAddedDuration.TraceAsync(() => $"{logs.Count()} Popularities added in {duration.TotalSeconds}sec"))
                .TryCatch()
                .OnFailure((toSave, unitResult) => this.LogError(unitResult.Exception, () => $"Could not bulk import track popularity"))
                .AsActionBlock(new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = AddTrackPopularitiesConcurrentThreadsCount, EnsureOrdered = false });
            #endregion addPlaylistTrackPopularitiesBlock

            var playlistWithTracksBroadcastBlock = new BroadcastBlock<Result<SpotifyPlaylistToSaveWithTracks>>(i => i);

            var updateFullPlaylistBatchBlock = new BatchBlock<Result<SpotifyPlaylistToSaveWithTracks>>(100);

            #region updateFullPlaylistBlock
            Func<IEnumerable<SpotifyPlaylist>, Task> fullPlaylistsUpdateAsync = _spotifyPlaylistManager.UpdatePlaylistsAsync;

            ActionBlock<IEnumerable<Result<SpotifyPlaylistToSaveWithTracks>>> updateFullPlaylistBlock =
            fullPlaylistsUpdateAsync
                .ToUnit()
                .Timeout(TimeSpan.FromSeconds(20))
                .Retry(2)
                .Duration((playlists, unit, duration) => _fullUpdateDuration.TraceAsync(() => $"Playlists ({playlists.Count()}) full updated in {duration.TotalSeconds}sec"))
                .Map<IEnumerable<Result<SpotifyPlaylistToSaveWithTracks>>, Unit, IEnumerable<SpotifyPlaylist>, Unit>(
                     inputTransform: toSaves => toSaves.Select(p => p.Value.Playlist),
                     outputTransform: (playlists, unit, toSaves) => Unit.Default
                )
                .TryCatch()
                .OnFailure((toSaves, unitResult) => this.LogError(unitResult.Exception, () => $"Could NOT fully update playlists."))
                .OnSuccess((toSaves, unitResult) =>
                {
                    var builder = new StringBuilder();
                    foreach (var playlist in toSaves)
                    {
                        playlist.Value.SpotifyPlaylistToSave.Input.PlaylistMarker.StageComplete(Stages.SavePlaylist);
                        builder.AppendLine($"888888888888 Playlist updated '{playlist.Value.SpotifyPlaylistToSave.IndexToString()}' Duration: {playlist.Value.Playlist.Duration} Followers: {playlist.Value.Playlist.Followers} TrackCount: {playlist.Value.Playlist.TrackCount} TrackLatestAdded: {playlist.Value.Playlist.TrackLatestAdded}");
                    }

                    _loggerFull.InfoAsync(builder.ToString);
                    _loggerCurrentProcessed.InfoAsync(builder.ToString);
                })
                .OnSuccess((toSaveResults, unitResult) => { progressTracker.NextAsync(toSaveResults.Count()).FireAndForget(); })
                .AsActionBlock(new ExecutionDataflowBlockOptions() { EnsureOrdered = false, MaxDegreeOfParallelism = FullUpdateConcurrentThreadsCount });
            #endregion updateFullPlaylistBlock

            #region softRemovePlaylistBlock
            Func<SpotifyPlaylistToSave, Task> softRemovePlaylistAsync = async toSave =>
            {
                //ApolloUpdatePlaylistInfoEventSource.Log.RemovePlaylistStart(toSave.SpotifyPlaylistData.PlaylistId);
                //if (toSave.Input.EditorialPlaylists != null && toSave.Input.EditorialPlaylists.Any())
                //{
                //    await RemoveEditorialPlaylistsAsync(toSave.Input.EditorialPlaylists);
                //}

                //await _spotifyPlaylistManager.DeletePlaylistByIdAsync(toSave.SpotifyPlaylistData.PlaylistId);
                //ApolloUpdatePlaylistInfoEventSource.Log.RemovePlaylistStop(toSave.SpotifyPlaylistData.PlaylistId);
            };

            ActionBlock<Result<SpotifyPlaylistToSave>> softRemovePlaylistBlock =
                softRemovePlaylistAsync
                .ToUnit()
                .Timeout(TimeSpan.FromSeconds(5))
                .Retry(1)
                .TryCatch()
                .OnFailure((toSave, unitResult) => this.LogError(unitResult.Exception, () => $"Could not remove playlist '{toSave.SpotifyPlaylistData.PlaylistId}'"))
                .OnSuccess((toSave, unitResult) =>
                {
                    toSave.Input.PlaylistMarker.StageComplete(Stages.RemovePlaylist);
                    _loggerFull.WarnAsync(() => $"----------- Removed playlist {toSave.SpotifyPlaylistData.PlaylistId}");
                })
                .OnSuccess((toSave, unitResult) => { progressTracker.NextAsync(1).FireAndForget(); })
                .AcceptResult()
                .AsActionBlock(new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 1, SingleProducerConstrained = false, EnsureOrdered = false });
            #endregion softRemovePlaylistBlock

            var propagetCompletionLinkOptions = new DataflowLinkOptions() { PropagateCompletion = true };
            var noPropagationLinkOptions = new DataflowLinkOptions() { PropagateCompletion = false };

            copyTracklistHistoryBlock.LinkTo(playlistToSaveBroadcastBlock, propagetCompletionLinkOptions, dto => dto.IsOk);
            copyTracklistHistoryBlock.LinkTo(DataflowBlock.NullTarget<Result<SpotifyPlaylistToSave>>());

            playlistToSaveBroadcastBlock.LinkTo(updateDatesBatchBlock, propagetCompletionLinkOptions);
            playlistToSaveBroadcastBlock.LinkTo(getPlaylistLatestTrackPopularitiesBlock, propagetCompletionLinkOptions);

            getPlaylistLatestTrackPopularitiesBlock.LinkTo(trackPopularitiesBatchBlock, noPropagationLinkOptions, dto => dto.IsOk);
            getPlaylistLatestTrackPopularitiesBlock.LinkTo(DataflowBlock.NullTarget<Result<SpotifyTrackPopularityLogValue>>());

            updateDatesBatchBlock.LinkTo(updatePlaylistsDatesBlock, propagetCompletionLinkOptions);

            updateEditorialPlaylistsBlock.LinkTo(playlistWithTracksBroadcastBlock, propagetCompletionLinkOptions, dto => dto.IsOk);
            updateEditorialPlaylistsBlock.LinkTo(DataflowBlock.NullTarget<Result<SpotifyPlaylistToSaveWithTracks>>());

            playlistWithTracksBroadcastBlock.LinkTo(updateFullPlaylistBatchBlock, propagetCompletionLinkOptions);
            playlistWithTracksBroadcastBlock.LinkTo(playlistToTrackPopularitiesBlock, propagetCompletionLinkOptions);

            playlistToTrackPopularitiesBlock.LinkTo(trackPopularitiesBatchBlock, noPropagationLinkOptions);

            trackPopularitiesBatchBlock.LinkTo(addPlaylistTrackPopularitiesBlock, propagetCompletionLinkOptions);

            updateFullPlaylistBatchBlock.LinkTo(updateFullPlaylistBlock, propagetCompletionLinkOptions);

            playlistSaveRelatedDataAndAddTracklistHistoryBlock.LinkTo(updateEditorialPlaylistsBlock, propagetCompletionLinkOptions);

            #region General shema diagram
            /*

           -------------------------------------------------
            Snapshot changed | Save tracklist | Save history|
           -------------------------------------------------
                   0		 |		0		  |  	0		|      ----------------------       -----------------
                             |				  |				|----->|Broadcast playlist  | ----> |Update playlist|
                   0		 |		0		  |		1		|      ----------------------       |update date    |
                             |				  |				|	        ^           \            -----------------
                   0		 |		1		  |		0		|           |            \
                             |				  |				|		    |             \        ---------------
            _________________|________________|_____________|           |              \     | Read playlist |
                             |				  |				|	   ------------------   \->  | latest track  |
                             |				  |				|----->|Copy tracklist  |        | popularities  |
                   0		 |		1		  |		1		|	   |   history		|        ----------------
                             |				  |				|	   ------------------               |
            _________________|________________|_____________|______________________                 \/
                             |				  |				|				                 ---------------            --------------
                             |				  |				|				                 | Batch track  |          | Add track   |
                   1		 |		0		  |  	0		|  			  |--------------->  | popularities | -------> | popularities|
                             |				  |				|		      |                  ----------------          -------------
                   1		 |		0		  |		1		|      ------------------
                             |				  |				|----->|Update editorial|       ----------------
                             |				  |				|      |  playlists     | ----> |Full playlist | 
                             |				  |				|      ------------------		|	update     |
                             |				  |				|           ^                   ----------------
                             |				  |				|			|
            _________________|________________|_____________|			| 
                   1		 |		1		  |		0		|       --------------- 
                             |				  |				|------>|Related data |
                             |				  |				|       |  update     |
                             |				  |				|       ---------------
            _________________|________________|_____________|				^ 
                   1		 |		1		  |		1		|        ----------------
                             |				  |				|------->|Add tracklist |
                             |				  |				|		 |  history     |
            _________________|________________|_____________|		 ----------------		  
            */
            #endregion

            spotifyPlaylistLoaderBlock.LinkTo(softRemovePlaylistBlock, propagetCompletionLinkOptions, dto => dto.IsOk && dto.Value.SpotifyPlaylistData.PlaylistRemoved);

            spotifyPlaylistLoaderBlock.LinkTo(copyTracklistHistoryBlock, propagetCompletionLinkOptions, dto => dto.IsOk && !dto.Value.SpotifyPlaylistData.PlaylistRemoved && !dto.Value.HasSnapshotIdChanged && dto.Value.SaveTracklist && dto.Value.SaveTracklistHistory);

            spotifyPlaylistLoaderBlock.LinkTo(playlistToSaveBroadcastBlock, noPropagationLinkOptions, dto => dto.IsOk && !dto.Value.SpotifyPlaylistData.PlaylistRemoved && !dto.Value.HasSnapshotIdChanged && !(dto.Value.SaveTracklist && dto.Value.SaveTracklistHistory));

            spotifyPlaylistLoaderBlock.LinkTo(updateEditorialPlaylistsBlock, noPropagationLinkOptions, dto => dto.IsOk && !dto.Value.SpotifyPlaylistData.PlaylistRemoved && dto.Value.HasSnapshotIdChanged && !dto.Value.SaveTracklist);

            spotifyPlaylistLoaderBlock.LinkTo(playlistSaveRelatedDataAndAddTracklistHistoryBlock, propagetCompletionLinkOptions, dto => dto.IsOk && !dto.Value.SpotifyPlaylistData.PlaylistRemoved && dto.Value.HasSnapshotIdChanged && dto.Value.SaveTracklist);
            spotifyPlaylistLoaderBlock.LinkTo(DataflowBlock.NullTarget<Result<SpotifyPlaylistToSave>>(), propagetCompletionLinkOptions);

            Stopwatch swTotal = new Stopwatch();
            swTotal.Start();

            const int MaxInputPostphone = 5;
            int currentPostphone = 0;
            bool errorExit = false;

            var cts = new CancellationTokenSource();

            var blockTrackerThread = TaskHelper.CreateLongRunningLoggingThread(
                cts.Token,
                () => String.Join(Environment.NewLine,
                       $"spotifyPlaylistLoaderBlock:                            {spotifyPlaylistLoaderBlock.InputCount}/{spotifyPlaylistLoaderBlock.OutputCount}/{spotifyPlaylistLoaderBlock.Completion.Status}",
                       $"copyTracklistHistoryBlock:                             {copyTracklistHistoryBlock.InputCount}/{copyTracklistHistoryBlock.OutputCount}/{copyTracklistHistoryBlock.Completion.Status}",
                       $"getPlaylistLatestTrackPopularitiesBlock:               {getPlaylistLatestTrackPopularitiesBlock.InputCount}/{getPlaylistLatestTrackPopularitiesBlock.OutputCount}/{getPlaylistLatestTrackPopularitiesBlock.Completion.Status}",
                       $"updateEditorialPlaylistsBlock:                         {updateEditorialPlaylistsBlock.InputCount}/{updateEditorialPlaylistsBlock.OutputCount}/{updateEditorialPlaylistsBlock.Completion.Status}",
                       $"playlistSaveRelatedDataAndAddTracklistHistoryBlock:    {playlistSaveRelatedDataAndAddTracklistHistoryBlock.InputCount}/{playlistSaveRelatedDataAndAddTracklistHistoryBlock.OutputCount}/{playlistSaveRelatedDataAndAddTracklistHistoryBlock.Completion.Status}",
                       $"softRemovePlaylist:                                    {softRemovePlaylistBlock.InputCount}/{softRemovePlaylistBlock.Completion.Status}",
                       $"updatePlaylistsDatesBlock:                             {updatePlaylistsDatesBlock.InputCount}/{updatePlaylistsDatesBlock.Completion.Status}",
                       $"updateFullPlaylistBlock:                               {updateFullPlaylistBlock.InputCount}/{updateFullPlaylistBlock.Completion.Status}",
                       $"addPlaylistTrackPopularitiesBlock:                     {addPlaylistTrackPopularitiesBlock.InputCount}/{addPlaylistTrackPopularitiesBlock.Completion.Status}",
                       Environment.NewLine),
                this.blockCountTracker,
                TimeSpan.FromSeconds(20));

            var progressChartThread = TaskHelper.CreateLongRunningThread(
                cts.Token,
                () => GenerateProgressChart(progressTracker, progressChartFilePath),
                TimeSpan.FromMinutes(5));

            foreach (var playlist in input.Playlists)
            {
                if (currentPostphone >= MaxInputPostphone)
                {
                    errorExit = true;
                    break;
                }

                var sendNewPlaylistCancellationToken = new CancellationTokenSource(TimeSpan.FromMinutes(2));
                try
                {
                    await spotifyPlaylistLoaderBlock.SendAsync(playlist, sendNewPlaylistCancellationToken.Token);
                    currentPostphone = 0;
                }
                catch (Exception ex)
                {
                    currentPostphone++;

                    this.LogError(ex, () => $"Could not send playlist to pipeline. PlaylistId: {playlist.PlaylistMarker.PlaylistId} Index: {playlist.PlaylistMarker.ProcessIndex}");
                    ErrorLoggingManager.Instance.LogError(ex);
                }
            }

            _loggerFull.WarnAsync(() => "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! INPUT POST COMPLETED !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!").FireAndForget();
            currentPostphone = 0;

            spotifyPlaylistLoaderBlock.Complete();

            if (!errorExit)
            {
                await Task.WhenAny(
                    Task.WhenAll(
                        softRemovePlaylistBlock.Completion,
                        updatePlaylistsDatesBlock.Completion,
                        updateFullPlaylistBlock.Completion),
                    Task.Delay(TimeSpan.FromMinutes(30.0)));
            }

            trackPopularitiesBatchBlock.Complete();
            await addPlaylistTrackPopularitiesBlock.Completion;

            cts.Cancel();

            var incompletePlaylistsCount = input.Playlists.Count(pl => pl.PlaylistMarker.IsFullyProcessed() == false);
            if (incompletePlaylistsCount > 0)
            {
                this.LogError(() => $"INCOMPLETED PLAYLISTS ({incompletePlaylistsCount})");
            }

            swTotal.Stop();
            _loggerCurrentProcessed.WarnAsync(() => $"UPDATED PLAYLISTS. TOTAL PROCESS TIME: {swTotal.Elapsed}").FireAndForget();

            await Task.WhenAll(blockTrackerThread, progressChartThread);

            await this.blockCountTracker.CompleteAsync();

            GenerateProgressChart(progressTracker, progressChartFilePath);

            await Task.WhenAll(
                this._loggerFull.Complete(),
                this._loggerCurrentProcessed.Complete(),
                this._loggerPlaylistLoad.Complete(),
                this._loggerErrors.Complete(),
                this._copyTracklistHistoryDuration.Complete(),
                this._playlistDatesDuration.Complete(),
                this._playlistEditorialsDuration.Complete(),
                this._playlistRelatedDataDuration.Complete(),
                this._trackPopularitiesAddedDuration.Complete(),
                this._fullUpdateDuration.Complete(),
                this._loggerGetLatestPopularitiesDuration.Complete(),
                this._loggerTracklistChanges.Complete(),
                this._loggerPersonalized.Complete()
            );
        }

        protected override void LogErrorOnly(Exception ex, Func<string> getMessage)
        {
            this._loggerFull.ErrorAsync(ex, getMessage).FireAndForget();
            this._loggerErrors.ErrorAsync(ex, getMessage).FireAndForget();
        }

        private async Task SaveRelatedDataAsync(SpotifyPlaylistInputData input, (SpotifyPlaylistToSave PlaylistInput, List<UpdatePlaylistTrack> Tracks, SpotifyPlaylist Playlist) spotifyApiData)
        {
            var playlistId = spotifyApiData.PlaylistInput.SpotifyPlaylistData.PlaylistId;

            if (ShouldCheckForPersonalized)
            {
                bool isPlaylistCurrentlyPersonalized = await DetermineAndUpdatePersonalizedSpotifyDataAsync(spotifyApiData.Playlist, spotifyApiData.Tracks, input);

                if (isPlaylistCurrentlyPersonalized)
                {
                    var personalizedTracks = await this._spotifyPlaylistManager.GetPlaylistPersonalizedTracksAsync(playlistId, spotifyApiData.Tracks.Select(x => x.Track.Isrc).ToList());
                    spotifyApiData.Tracks.AddRange(personalizedTracks);
                }
            }

            List<UpdatePlaylistTrack> existingTracklist = await this._spotifyPlaylistManager.GetTrackListAsync(playlistId);
            await _updatePlaylistsHelper.SetEarliestAddedDateAsync(spotifyApiData.Tracks, existingTracklist, playlistId, spotifyApiData.PlaylistInput.SpotifyPlaylistData.Username);

            if (_updatePlaylistsHelper.IsTrackListChanged(existingTracklist, spotifyApiData.Tracks))
            {
                await ((Func<string, IEnumerable<UpdatePlaylistTrack>, string, Task>)_spotifyPlaylistManager.SetSpotifyTrackListAndUpdateStatistics)
                   .Tuple()
                   .ToUnit()
                   .Retry(2)
                   .OnSuccessUnsafe((tp, unit) => _loggerTracklistChanges.WarnAsync(() => $"Updated playlist current tracklist and statistics for '{tp.Item1}'").FireAndForget())
                   .OnFailureWithRethrow((tp, result) =>
                   {
                       _loggerTracklistChanges.ErrorAsync(result.Exception, () => $"Could not update playlist current tracklist or statistics. PlaylistId: {tp.Item1} Error: {result.Exception.Message} Duration: {result.Duration.TotalSeconds}sec. ").FireAndForget();
                       this.LogError(result.Exception, () => $"SetSpotifyTrackListAndUpdateStatistics. PlaylistId: {tp.Item1} Duration: {result.Duration.TotalSeconds}secs");
                   })
                   .UnwrapTuple()
                   (playlistId, 
                    spotifyApiData.Tracks, 
                    String.IsNullOrWhiteSpace(spotifyApiData.Playlist.CountryCode) ? Constants.Spotify.DefaultPlaylistCountryCodeForStatistics : spotifyApiData.Playlist.CountryCode);
            }
            else
            {
                await _spotifyPlaylistManager.UpdateCurrentTracklistForTodayAsync(spotifyApiData.Playlist.PlaylistId);
                _loggerTracklistChanges.WarnAsync(() => $"{spotifyApiData.Playlist.PlaylistId} tracklist did not change").FireAndForget();
            }

            if (spotifyApiData.PlaylistInput.SaveTracklistHistory)
            {
                if (await _historicTrackListManager.DoesPlaylistHaveHistoryForDate(playlistId, DateTime.UtcNow) == false)
                {
                    await _historicTrackListManager.AddSpotifyTrackListHistoryAsync(playlistId, spotifyApiData.Tracks, DateTime.UtcNow);
                }
            }

            await AddTracksWithRelatedDataAsync(spotifyApiData.Tracks);

            await this.SavePlaylistChangesAsync(spotifyApiData.Playlist, this._spotifyPlaylistManager.GetPlaylistById(playlistId));
        }

        private void FixPlaylistBuzzCategoryAndCountry(SpotifySpecificData playlist, string dbPlaylistCountryCode)
        {
            BuzzUser buzzUser = base.GetBuzzUserByNameOrDefault(playlist.Username);
            playlist.BuzzCategoryId = buzzUser?.BuzzCategoryId;

            playlist.Country = GetCountry(dbPlaylistCountryCode, buzzUser);
        }

        private async Task RemoveEditorialPlaylistsAsync(IEnumerable<EditorialPlaylist> spotifyPlaylistGroup)
        {
            foreach (var editorialPlaylist in spotifyPlaylistGroup)
            {
                await _editorialPlaylistManager.DeleteEditorialPlaylistAsync(editorialPlaylist);

                _loggerFull.InfoAsync(() => $"Removed editorial playlist {editorialPlaylist.ID}, Spotify playlist Id: {editorialPlaylist.SpotifyLink.Uri}").FireAndForget();
            }
        }

        private async Task UpdateEditorialPlaylistsAsync((IEnumerable<EditorialPlaylist> EditorialPlaylists, SpotifySpecificData Playlist) tuple)
        {
            if (tuple.EditorialPlaylists != null && tuple.EditorialPlaylists.Any())
            {
                ApolloUpdatePlaylistInfoEventSource.Log.UpdateEditorialPlaylistsParallelStart(tuple.Playlist.PlaylistId);
                foreach (var editorialPlaylist in tuple.EditorialPlaylists)
                {
                    try
                    {
                        var updatedPlaylist = await UpdateEditorialPlaylistAsync(tuple.Playlist, editorialPlaylist);
                    }
                    catch (Exception ex)
                    {
                        this.LogError(ex, () => $"Could not save editorial playlist {editorialPlaylist.Name}");
                    }
                }
                ApolloUpdatePlaylistInfoEventSource.Log.UpdateEditorialPlaylistsParallelStop(tuple.Playlist.PlaylistId);
            }
        }

        private async Task<EditorialPlaylist> UpdateEditorialPlaylistAsync(SpotifySpecificData spotifyPlaylist, EditorialPlaylist editorialPlaylist)
        {
            //TODO: Check for changes and only run update if necessary.

            editorialPlaylist.Name = spotifyPlaylist.Name;
            if (string.IsNullOrWhiteSpace(editorialPlaylist.DisplayName))
            {
                editorialPlaylist.DisplayName = spotifyPlaylist.Name;
            }

            editorialPlaylist.CountryCode = spotifyPlaylist.Country;
            editorialPlaylist.BuzzCategoryId = spotifyPlaylist.BuzzCategoryId;
            editorialPlaylist.OwnerUsername = spotifyPlaylist.Username;

            editorialPlaylist.SpotifyDescription = spotifyPlaylist.Description;
            if (spotifyPlaylist.SpotifyImageFilename != null)
            {
                editorialPlaylist.SpotifyImageFileName = spotifyPlaylist.SpotifyImageFilename;
            }

            await _editorialPlaylistManager.UpdateSpotifyFieldsAsync(editorialPlaylist);

            return editorialPlaylist;
        }

        protected virtual async Task<(PlaylistResponse Response, RequestData Data, IEnumerable<KeyValuePair<string, string>> RawResponses)> GetPlaylistWithTracksAsync(string playlistId, string fields, string snapshotId)
        {
            return (await _spotifyWebApi.GetPlaylistByIdWithAllTracksHavingSnapshotIdAsync(playlistId, snapshotId, fields), null, null);
        }

        private async Task<SpotifySpecificData> GetSpotifyPlaylistSpecificDataAsync(string playlistId, string snapshotId)
        {
            Stopwatch sw = new Stopwatch();

            sw.Start();

            var playlistData = await this.GetPlaylistWithTracksAsync(playlistId, Constants.Spotify.VendorPlaylistWithTracksFullFieldsList, snapshotId);

            PlaylistResponse spotifyPlaylist = playlistData.Response;

            if (spotifyPlaylist == null)
            {
                return new SpotifySpecificData()
                {
                    PlaylistId = playlistId,
                    PlaylistRemoved = true
                };
            }
            
            spotifyPlaylist.RemoveInvalidTracks();

            double playlistWithTracksLoadMilliseconds = sw.Elapsed.TotalMilliseconds;

            string spotifyImageFileName = null;

            StringBuilder playlistLoadLogMessage = new StringBuilder($"Playlist '{playlistId}' with all {spotifyPlaylist.Tracks.total} tracks loaded {playlistWithTracksLoadMilliseconds} ms. HasShapshotIdChanged {spotifyPlaylist.HasSnapshotIdChanged}(OLD '{snapshotId}' NEW '{spotifyPlaylist.snapshot_id}') Vendor Cache: {ApolloCache}");

            if (spotifyPlaylist.HasSnapshotIdChanged && SavePlaylistImageToAmazon)
            {
                sw.Restart();
                try
                {
                    spotifyImageFileName = await _spotifyImageHandler.StoreLargestPlaylistImageAsync(playlistId, spotifyPlaylist.images?.FirstOrDefault()?.url);
                }
                catch (Exception ex)
                {
                    this.LogError(ex, () => $"Error saving image for playlist {playlistId}");
                }
                sw.Stop();
                playlistLoadLogMessage.Append($" Image loaded: {sw.Elapsed.TotalMilliseconds}ms. Total: {playlistWithTracksLoadMilliseconds + sw.Elapsed.TotalMilliseconds}ms");
            }

            if (playlistData.Data != null)
            {
                playlistLoadLogMessage.Append($" {playlistData.Data.Message}");
            }

            _loggerPlaylistLoad.DebugAsync(playlistLoadLogMessage.ToString).FireAndForget();

            return new SpotifySpecificData()
            {
                SpotifyLink = new SpotifyLink($"spotify:playlist:{SpotifyLink.ExtrackPlaylistID(spotifyPlaylist.uri)}"),
                PlaylistId = playlistId,
                Tracks = spotifyPlaylist.Tracks.items,
                TotalTracks = spotifyPlaylist.Tracks.total,
                Description = spotifyPlaylist.description,
                Name = spotifyPlaylist.name,
                SpotifyImageFilename = spotifyImageFileName,
                Username = spotifyPlaylist.owner?.id,
                Public = spotifyPlaylist.@public,
                HasSnapshotIdChanged = spotifyPlaylist.HasSnapshotIdChanged,
                SnapshotId = spotifyPlaylist.snapshot_id
            };
        }



        private class SpotifySpecificData
        {
            public SpotifyLink SpotifyLink { get; set; }
            public string PlaylistId { get; set; }
            public string Name { get; set; }
            public string Description { get; set; }
            public string Country { get; set; }

            public bool PlaylistRemoved { get; set; }

            public IEnumerable<PlaylistTrackItem> Tracks { get; set; }

            public string SpotifyImageFilename { get; set; }
            public int? BuzzCategoryId { get; set; }
            public string Username { get; set; }

            public bool Public { get; set; }

            public DateTime ProcessStartTime { get; set; }

            public Contracts.Entities.SpotifyPlaylist ToDbEntity(List<UpdatePlaylistTrack> tracks, DateTime? trackLatestAdded)
            {
                return new Contracts.Entities.SpotifyPlaylist()
                {
                    PlaylistId = this.PlaylistId,
                    PlaylistUri = this.SpotifyLink.Uri,
                    CountryCode = this.Country,
                    Description = this.Description,
                    Name = this.Name,
                    User = this.Username,
                    TrackCount = this.TotalTracks,
                    Duration = tracks.Sum(t => (long)t.Track.Duration),
                    BuzzCategoryId = this.BuzzCategoryId,
                    Image = this.SpotifyImageFilename,
                    TrackLatestAdded = trackLatestAdded,
                    Public = this.Public,
                    UpdateDate = DateTime.UtcNow,
                    SnapshotId = this.SnapshotId
                };
            }

            public string SnapshotId { get; set; }

            public bool HasSnapshotIdChanged { get; set; }

            public int TotalTracks { get; set; }
        }

        private class SpotifyPlaylistInputData
        {
            public List<SpotifyPlaylistInput> Playlists { get; set; }

            public ServiceAccount PersonalizedPlaylistsServiceAccount { get; set; }

            public HashSet<string> PersonalizedPlaylists { get; set; }
        }

        [Flags]
        public enum Stages
        {
            Init = 0,
            SpotifyLoad = 1,
            SaveRelated = 2,
            SavePlaylist = 4,
            RemovePlaylist = 8
        }

        public struct PlaylistMarker
        {
            public readonly string PlaylistId;
            public readonly string SnapshotId;
            public readonly string CountryCode;
            public readonly int ProcessIndex;

            private Stages CompletedStages;

            public PlaylistMarker(string playlistId, string snapshotId, string countryCode, int playlistIndex)
            {
                this.PlaylistId = playlistId;
                this.SnapshotId = snapshotId;
                this.ProcessIndex = playlistIndex;
                this.CountryCode = countryCode;
                this.CompletedStages = Stages.Init;
            }

            public void StageComplete(Stages stage)
            {
                this.CompletedStages |= stage;
            }

            public bool IsFullyProcessed()
            {
                if (((this.CompletedStages & Stages.SavePlaylist) == Stages.SavePlaylist) || ((this.CompletedStages & Stages.RemovePlaylist) == Stages.RemovePlaylist))
                {
                    return true;
                }

                return false;
            }

            public override string ToString()
            {
                return $"PlaylistMarker->PlaylistId {this.PlaylistId} Index {this.ProcessIndex}";
            }
        }

        internal struct SpotifyPlaylistInput
        {
            public readonly PlaylistMarker PlaylistMarker;
            public readonly IReadOnlyList<EditorialPlaylist> EditorialPlaylists;
            public readonly bool SaveTracklist;

            public SpotifyPlaylistInput(PlaylistMarker playlistMarker, IReadOnlyList<EditorialPlaylist> editorialPlaylists, bool saveTracklist)
            {
                this.PlaylistMarker = playlistMarker;
                this.EditorialPlaylists = editorialPlaylists;
                this.SaveTracklist = saveTracklist;
            }

            public override string ToString()
            {
                return $"SpotifyPlaylistinput->PlaylistId {this.PlaylistMarker.PlaylistId} Index {this.PlaylistMarker.ProcessIndex}->Editorial {this.EditorialPlaylists != null && this.EditorialPlaylists.Any()}->SaveTracklist: {this.SaveTracklist}";
            }
        }

        private struct SpotifyPlaylistToSave
        {
            public readonly SpotifyPlaylistInput Input;
            public readonly SpotifySpecificData SpotifyPlaylistData;

            public readonly bool SaveTracklistHistory;
            public bool HasSnapshotIdChanged { get { return this.SpotifyPlaylistData.HasSnapshotIdChanged; } }
            public bool SaveTracklist { get { return this.Input.SaveTracklist; } }

            public SpotifyPlaylistToSave(SpotifyPlaylistInput input, SpotifySpecificData spotifyPlaylistData, bool shouldSaveTrackListHistory)
            {
                this.Input = input;
                this.SpotifyPlaylistData = spotifyPlaylistData;
                this.SaveTracklistHistory = shouldSaveTrackListHistory;
            }

            public override string ToString()
            {
                return $"{this.IndexToString()}->Removed {this.SpotifyPlaylistData?.PlaylistRemoved}->SaveTracklistHistory {this.SaveTracklistHistory}->SnapshotChanged {this.HasSnapshotIdChanged}->SaveTrackList {this.SaveTracklist}";
            }

            public string IndexToString()
            {
                return $"PlaylistId {this.Input.PlaylistMarker.PlaylistId} Index {this.Input.PlaylistMarker.ProcessIndex}";
            }
        }

        private struct SpotifyPlaylistToSaveWithTracks
        {
            public readonly SpotifyPlaylistToSave SpotifyPlaylistToSave;
            public readonly IEnumerable<UpdatePlaylistTrack> Tracks;
            public readonly Sony.Filtr.Contracts.Entities.SpotifyPlaylist Playlist;

            public SpotifyPlaylistToSaveWithTracks(SpotifyPlaylistToSave toSave,
                Contracts.Entities.SpotifyPlaylist playlist,
                IEnumerable<UpdatePlaylistTrack> tracks)
            {
                this.Tracks = tracks;
                this.Playlist = playlist;
                this.SpotifyPlaylistToSave = toSave;
            }

            public override string ToString()
            {
                return base.ToString() + $"Tracks {this.Tracks?.Count()}";
            }
        }

        private void ClearCacheForBuzzImportUsers()
        {
            var users = _buzzManager.GetBuzzImportUsers();
            foreach (var buzzImportUser in users)
            {
                _buzzManager.ClearBuzzImportUserCache(buzzImportUser);
            }

            _buzzManager.ClearTopBuzzCompaniesCache();
        }
    }

    public class RequestData
    {
        public readonly string Message;

        public RequestData(string message)
        {
            this.Message = message;
        }
    }
}