using System; using System.Collections.Concurrent; using System.Collections.Generic; using System.Globalization; using System.IO; using System.IO.Compression; using System.Linq; using System.Threading; using System.Threading.Tasks; using System.Threading.Tasks.Dataflow; using Amazon.S3; using Amazon.S3.Model; using Amazon.S3.Transfer; using MoreLinq; using MySql.Data.MySqlClient; using Newtonsoft.Json; using NLog; using PetaPoco.Business; using Sony.Filtr.ApolloAPI; using Sony.Filtr.ApolloAPI.Models.Apple; using Sony.Filtr.AppleMusic; using Sony.Filtr.AppleMusic.Data.Streams; using Sony.Filtr.AppleMusic.Playlists; using Sony.Filtr.AppleMusic.Streams; using Sony.Filtr.Contracts.Abstractions; using Sony.Filtr.Contracts.Definitions; using Sony.Filtr.Functional; using Sony.Filtr.Playlists; using Sony.Filtr.Utility; using Sony.Filtr.Utility.Extensions; using Sony.Filtr.VendorToSpotify; namespace Sony.Filtr.Tasks.Tasks.AppleMusic { public class UpdateAppleMusicAnalyticsAggregatedTask : IScheduledTask { private readonly AppleMusicStreamingAggregatedReportApi _aggregatedReportApi; private readonly AppleMusicPlaylistManager _appleMusicPlaylistManager; private readonly AppleMusicStreamsManager _appleMusicStreamsManager; private readonly IApolloAppleWebApi _apolloAppleApi; private readonly Logger _logger; private const string DataFileFolderName = "appleMusicAnalytics"; public UpdateAppleMusicAnalyticsAggregatedTask( AppleMusicStreamingAggregatedReportApi aggregatedReportApi, AppleMusicPlaylistManager appleMusicPlaylistManager, AppleMusicStreamsManager appleMusicStreamsManager, IApolloAppleWebApi apolloAppleApi) { _aggregatedReportApi = aggregatedReportApi; _appleMusicPlaylistManager = appleMusicPlaylistManager; _appleMusicStreamsManager = appleMusicStreamsManager; _apolloAppleApi = apolloAppleApi; _logger = LogManager.GetLogger("UpdateAppleMusicAnalyticsAggregatedTask"); } private ConcurrentDictionary existingPlaylistIds = null; private static AppleS3FileType[] supportedFileTypes = new AppleS3FileType[] { //AppleS3FileType.Demographics, //AppleS3FileType.StreamsTracks, //AppleS3FileType.StreamsContainerTracks, AppleS3FileType.StreamsContainers, }; public async Task ExecuteAsync(Guid scheduledTaskLogId) { var taskLog = new ScheduledTaskLog(); if (!Directory.Exists(DataFileFolderName)) { Directory.CreateDirectory(DataFileFolderName); } int newStreamsDetectedFlag = 0; existingPlaylistIds = new ConcurrentDictionary((await _appleMusicPlaylistManager.GetExistingPlaylistIdsAsync()).ToDictionary(str => str, str => true)); TransformManyBlock> downloadDateFilesBlock = ((Func>>)GetFilesToProcessForDate) .Timeout(TimeSpan.FromSeconds(10)) .Retry(1) .TryCatch() .OnSuccess((dt, result) => _logger.Debug(() => $"For date '{dt.ToShortDateString()}' downloaded {result.Value.Count()} files")) .OnFailure((dt, result) => _logger.Error(result.Exception, $"Could not download files for date '{dt.ToShortDateString()}'")) .SelectMany() .AsTransformManyBlock(new ExecutionDataflowBlockOptions() { MaxDegreeOfParallelism = FileProcessThreadsCount(), EnsureOrdered = false }); TransformBlock, Result<(AppleMusicAggregatedS3File, AppleMusicContainerStream[])>> downloadContainerStreamFileBlock = ((Func>)DownloadAppleMusicContainerStreamFile) .Timeout(TimeSpan.FromSeconds(10)) .Retry(1) .Map( inputTransform: x => x, outputTransform: (f, streams, f2) => (f, streams)) .TryCatch() .OnSuccess((file, tupleResult) => _logger.Debug(() => $"File Date: '{file.Date.ToShortDateString()}' '{file.FilePath}' has {tupleResult.Value.Item2.Length} items")) .OnFailure((file, result) => _logger.Error(result.Exception, $"Could not load content for Date: '{file.Date.ToShortDateString()}' '{file.FilePath}'")) .AcceptResult() .AsTransformBlock(new ExecutionDataflowBlockOptions() { MaxDegreeOfParallelism = DownloadFilesThreadsCount(), EnsureOrdered = false }); TransformBlock, Result<(AppleMusicAggregatedS3File, AppleMusicContainerStream[])>> bulkLoadContainerStreamsBlock = ((Func)BulkLoadContainerStreams) .ToUnit() .Timeout(TimeSpan.FromMinutes(5)) .Retry(1) .Map, (AppleMusicAggregatedS3File, AppleMusicContainerStream[]), AppleMusicContainerStream[], Unit>( inputTransform: tuple => tuple.Value.Item2, outputTransform: (stream, stream2, result) => { Interlocked.Exchange(ref newStreamsDetectedFlag, 1); return result.Value; }) .TryCatch() .OnFailure((streams, result) => _logger.Error(result.Exception, $"Could not bulk load container streams")) .OnSuccess((blockInput, result) => _logger.Info(() => $"Bulk loaded {blockInput.Value.Item2.Length} streams from file '{blockInput.Value.Item1.FilePath}'")) .AsTransformBlock(new ExecutionDataflowBlockOptions() { MaxDegreeOfParallelism = 1, EnsureOrdered = false }); TransformBlock, Result> markAsProcessedBlock = ((Func)_appleMusicStreamsManager.MarkFileAsProcessedAsync) .Tuple() .ToUnit() .Timeout(TimeSpan.FromSeconds(10)) .Retry(1) .Map<(AppleMusicAggregatedS3File, AppleMusicContainerStream[]), AppleMusicContainerStream[], (DateTime, AppleMusicAggregatedS3File), Unit>( inputTransform: tuple => (tuple.Item1.Date, tuple.Item1), outputTransform: (tupleFunc, b, tupleInput) => tupleInput.Item2 ) .TryCatch() .OnSuccess((tupleInput, result) => _logger.Debug(() => $"Marked file as processed '{tupleInput.Item1.FilePath}'")) .OnFailure((tuple, result) => _logger.Error(result.Exception, $"Could not mark file as processed '{tuple.Item1.FilePath}'")) .AcceptResult() .AsTransformBlock(new ExecutionDataflowBlockOptions() { MaxDegreeOfParallelism = 1, EnsureOrdered = false }); var finalizer = new ActionBlock>(result => { }, new ExecutionDataflowBlockOptions() { MaxDegreeOfParallelism = 1 }); TransformManyBlock, string> filerOutExistingPlaylistsBlock = new TransformManyBlock, string>( result => FilterOutExistingPlaylistIds(result.Value), new ExecutionDataflowBlockOptions() { MaxDegreeOfParallelism = 1, EnsureOrdered = false }); TransformBlock> loadPlaylistFromAppleBlock = ((Func>)this.GetPlaylistFromAppleAsync) .Timeout(TimeSpan.FromSeconds(10)) .Retry(1) .TryCatch() .OnFailure((id, result) => _logger.Error(result.Exception, $"Failed to load playlist from Apple. id: '{id}'")) .OnSuccess((id, result) => { if (result.Value != null) { _logger.Debug(() => $"Loaded new Apple playlist. id: '{id}'"); } else { _logger.Debug(() => $"Apple does not know new playlist for any major storefront. id: '{id}'"); } }) .AsTransformBlock(new ExecutionDataflowBlockOptions() { MaxDegreeOfParallelism = 1, EnsureOrdered = false }); BatchBlock> addPlaylistBatchBlock = new BatchBlock>(1000); ActionBlock[]> savePlaylistsBlock = ((Func)this.BulkLoadApplePlaylists) .ToUnit() .Timeout(TimeSpan.FromSeconds(10)) .Retry(1) .Map[], Unit, ApplePlaylistData[], Unit>( inputTransform: blockInput => blockInput.Select(r => r.Value).ToArray(), outputTransform: (array, unit, result) => unit ) .TryCatch() .OnFailure((playlists, result) => _logger.Error(result.Exception, $"Could not save new playlists batch to DB.")) .OnSuccess((playlists, result) => _logger.Debug(() => $"Saved new playlists batch to DB.")) .AsActionBlock(new ExecutionDataflowBlockOptions() { MaxDegreeOfParallelism = 1, EnsureOrdered = false }); var linkOptions = new DataflowLinkOptions() { PropagateCompletion = true }; downloadDateFilesBlock.LinkTo(downloadContainerStreamFileBlock, linkOptions, r => r.IsOk); downloadDateFilesBlock.LinkTo(DataflowBlock.NullTarget>()); downloadContainerStreamFileBlock.LinkTo(bulkLoadContainerStreamsBlock, linkOptions, r => r.IsOk && r.Value.Item1.FileType == AppleS3FileType.StreamsContainers && r.Value.Item2.Any()); downloadContainerStreamFileBlock.LinkTo(DataflowBlock.NullTarget>()); bulkLoadContainerStreamsBlock.LinkTo(markAsProcessedBlock, linkOptions, r => r.IsOk); bulkLoadContainerStreamsBlock.LinkTo(DataflowBlock.NullTarget>()); markAsProcessedBlock.LinkTo(finalizer, linkOptions); //markAsProcessedBlock.LinkTo(filerOutExistingPlaylistsBlock, linkOptions, r => r.IsOk); //markAsProcessedBlock.LinkTo(DataflowBlock.NullTarget>()); filerOutExistingPlaylistsBlock.LinkTo(loadPlaylistFromAppleBlock, linkOptions); loadPlaylistFromAppleBlock.LinkTo(addPlaylistBatchBlock, linkOptions, r => r.IsOk); loadPlaylistFromAppleBlock.LinkTo(DataflowBlock.NullTarget>()); addPlaylistBatchBlock.LinkTo(savePlaylistsBlock, linkOptions); foreach (var date in new DateTime(2021, 1, 1).GetDateRangeTo(DateTime.Today).Reverse()) { await downloadDateFilesBlock.SendAsync(date); } downloadDateFilesBlock.Complete(); //await savePlaylistsBlock.Completion; await finalizer.Completion; _logger.Info("UpdateAppleMusicAnalyticsAggregatedTask file process finished."); Func shouldRecalculatePlaylistStreams = () => newStreamsDetectedFlag == 1; if (shouldRecalculatePlaylistStreams()) { await SetContainerStreamsSummaryAsync(); } return taskLog; } private async Task GetPlaylistFromAppleAsync(string playlistId) { foreach (var storefront in AppleMusicPlaylistManager.MajorStorefronts) { var playlist = await _apolloAppleApi.GetPlaylistAsync(storefront, playlistId); if (playlist != null) { return playlist; } } return null; } private async Task BulkLoadApplePlaylists(IEnumerable playlists) { await BulkLoader.LoadAsync( "tblAppleMusicPlaylist", new Func[] { p => p.id, p => p.attributes.name, p => p.attributes.playlistType, p => p.relationships?.curator?.data?.FirstOrDefault()?.id.ToString(), p => Boolean.FalseString }, new string[] { "Id", "Name", "PlaylistType", "CuratorId", "Removed" }, playlists, MySqlConnector.MySqlBulkLoaderConflictOption.Replace); } private async Task> GetFilesToProcessForDate(DateTime dt) { var filesTask = _aggregatedReportApi.GetFilesAsync(dt); var alreadyImportedFiles = PetaPocoRepository.ReadOnlyInstance.Fetch("WHERE date=@0", dt); await filesTask; return filesTask.Result .Where(p => supportedFileTypes.Contains(p.FileType)) .Where(p => !alreadyImportedFiles.Any(a => a.CountryCode == p.Country && a.DistributerId == p.DistributorId && a.VendorId == p.VendorId && a.FileType == p.FileType)) //.Take(2) .ToArray(); } private Task DownloadAppleMusicContainerStreamFile(AppleMusicAggregatedS3File file) { return DownloadFileContent(file); } private async Task DownloadFileContent(AppleMusicAggregatedS3File trackFile) where TParseToType : class, IAggregatedStreamFormat { var result = await Result.Success(trackFile) .Bind(f => (File: f, DatePart: f.Date.ToString("yyyy-dd-M--HH-mm-ss"))) .Bind(tuple => (LocalPath: Path.Combine(DataFileFolderName, $"track-{tuple.DatePart}-{tuple.File.DistributorId}-{tuple.File.VendorId}-{tuple.File.Country}-{tuple.File.Version}-{tuple.File.FileType}.ndjson"), SourcePath: tuple.File.FilePath, DistributorId: tuple.File.DistributorId)) .TapAsync(tuple => _aggregatedReportApi.DownloadToFileAsync(tuple.LocalPath, tuple.SourcePath)) .BindFinalize( func: tuple => (LocalPath: tuple.LocalPath, Contents: ReadNJsonData(tuple.LocalPath, tuple.DistributorId)), finalizer: tuple => { if (File.Exists(tuple.LocalPath)) { File.Delete(tuple.LocalPath); } }); return result.Flatten().Contents.ToArray(); } private Task BulkLoadContainerStreams(AppleMusicContainerStream[] streams) { return BulkLoadContent(streams); } private static Task BulkLoadContent(TContent[] items) { if (items.Any()) { PetaPocoRepository.Instance.ImportBulkFileLoader(items); } return Task.FromResult(items.Any()); } private List ReadNJsonData(string localPath, int trackFileDistributorId) where T : class, IAggregatedStreamFormat { List objects = new List(); var jsonSerializer = new JsonSerializer(); using (var fileStream = File.OpenRead(localPath)) { using (var jsonReader = new JsonTextReader(new StreamReader(fileStream)) { SupportMultipleContent = true }) { while (jsonReader.Read()) { try { var obj = jsonSerializer.Deserialize(jsonReader); obj.DistributerId = trackFileDistributorId; objects.Add(obj); } catch (Exception e) { _logger.Error(e); } } } } return objects; } private async Task SetContainerStreamsSummaryAsync() { _logger.Debug("UpdateAppleMusicAnalyticsAggregatedTask begin container stream summary recalculation"); var playlists = await _appleMusicPlaylistManager.GetPlaylistsAsync(); var containerIds = playlists.Items.Select(p => p.Id).ToList(); _logger.Debug($"Will calculate container summary for {containerIds.Count()} containers"); ConcurrentBag allContainerStreamsSummaries = new ConcurrentBag(); var batches = containerIds.Batch(10).ToList(); var latestDate = await _appleMusicStreamsManager.GetLatestDayWithAggregatedStreamsDataAsync(AppleS3FileType.StreamsContainers); await batches.ItemIndex().ForEachAsync(5, async playlistBatch => { Console.WriteLine($"Calculating for batch {playlistBatch.Index} / {batches.Count}"); var summaries = await _appleMusicStreamsManager.CalculateAppleMusicContainerStreamsSummariesAsync(playlistBatch.Item, latestDate); summaries.ForEach(s => allContainerStreamsSummaries.Add(s)); }); _logger.Debug($"Done calculating container summaries"); _logger.Debug($"Bulk loading to db..."); PetaPocoRepository.Instance.ImportBulkFileLoader(allContainerStreamsSummaries, MySqlConnector.MySqlBulkLoaderConflictOption.Replace); _logger.Debug("Done SetContainerStreamsSummaryAsync"); } private IEnumerable FilterOutExistingPlaylistIds(AppleMusicContainerStream[] streams) { var newIds = streams.Select(s => s.ContainerId).Where(id => id.ToLower().StartsWith("pl.")) .Except(existingPlaylistIds.Keys) .ToArray(); newIds.ForEach(str => existingPlaylistIds.AddOrUpdate(str, true, (s, b) => true)); return newIds; } private static int FileProcessThreadsCount() { return Maybe.GetAppSettingsIntOrDefault("UpdateAppleMusicAnalyticsAggregatedTask_File_Process_Threads_Count", 2); } private static int DownloadFilesThreadsCount() { return Maybe.GetAppSettingsIntOrDefault("UpdateAppleMusicAnalyticsAggregatedTask_Download_Files_Threads_Count", 2); } private static string NewPlaylistsFolderPath() { var path = Maybe.GetAppSettingsStringOrDefault("UpdateAppleMusicAnalyticsAggregatedTask_New_Playlist_Ids_File_Path", Environment.CurrentDirectory); if (String.IsNullOrEmpty(path)) { path = Environment.CurrentDirectory; } return path; } private async Task SaveNewPlaylistIdsAsync(string[] newPlaylistIds) { if (!newPlaylistIds.Any()) { return; } string folderPath = NewPlaylistsFolderPath(); if (!Directory.Exists(folderPath)) { _logger.Warn($"Specified folder for new playlist ids file does not exist '{folderPath}'. Defaulting to current folder '{Environment.CurrentDirectory}'"); folderPath = Environment.CurrentDirectory; } string filePath = Path.Combine(folderPath, $"New_Playlist_Ids_{DateTime.Now.ToString("yyyy_MM_dd")}_{Guid.NewGuid().ToString()}.txt"); using (StreamWriter outputFile = new StreamWriter(filePath)) { await outputFile.WriteAsync(string.Join(Environment.NewLine, newPlaylistIds)); } } } public class AppleMusicStreamingAggregatedReportApi { private readonly AmazonS3Client _s3Client; private const string _Bucket = "sme-aggregated-stream-reports"; public AppleMusicStreamingAggregatedReportApi(AmazonS3Client s3Client) { _s3Client = s3Client; } public async Task> GetFilesAsync(DateTime date) { List filenames = new List(); var dateString = date.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture); var streamObjects = await _s3Client.ListObjectsAsync(new ListObjectsRequest() { BucketName = _Bucket, Prefix = $"apple/{dateString}/", }); filenames.AddRange(streamObjects.S3Objects.Select(p => p.Key).ToList()); var nextMarker = streamObjects.NextMarker; while (!string.IsNullOrWhiteSpace(nextMarker)) { var moreResults = await _s3Client.ListObjectsAsync(new ListObjectsRequest() { BucketName = _Bucket, Prefix = $"apple/{dateString}/", Marker = nextMarker }); nextMarker = moreResults.NextMarker; filenames.AddRange(moreResults.S3Objects.Select(p => p.Key).ToList()); } var fileNames = filenames.Select(p => ParseAppleMusicS3FilePath(p)).Where(p => p != null).ToList(); return fileNames; } public async Task DownloadToFileAsync(string destinationFilepath, string fileKey) { using (var downloadStream = await DownloadS3FileAsync(_Bucket, fileKey)) { using (var fileStream = File.Open(destinationFilepath, FileMode.OpenOrCreate)) { using (var gunzippedStream = new GZipStream(downloadStream, CompressionMode.Decompress)) { await gunzippedStream.CopyToAsync(fileStream); } } } } private async Task DownloadS3FileAsync(string bucket, string key) { var transferUtility = new TransferUtility(_s3Client); var contentStream = await transferUtility.OpenStreamAsync(new TransferUtilityOpenStreamRequest() { BucketName = bucket, Key = key, }); return contentStream; } private AppleMusicAggregatedS3File ParseAppleMusicS3FilePath(string filename) { //apple_2017-12-05_theorchard_v1_85420853_streams_tracks.ndjson try { var parsedFilename = Path.GetFileNameWithoutExtension(filename); if (string.IsNullOrWhiteSpace(parsedFilename)) return null; var fileParts = parsedFilename.Split('_'); if (!fileParts.Any()) return null; if (fileParts.ElementAt(0) != "apple") return null; var date = fileParts.ElementAt(1); var distributorName = fileParts.ElementAt(2); var version = fileParts.ElementAt(3); var vendorId = int.Parse(fileParts.ElementAt(4)); string country = "global"; AppleS3FileType fileType = AppleS3FileType.Unknown; var type = fileParts.ElementAt(5); if (type == "streams") { var type2 = fileParts.ElementAt(6); if (type2 == "tracks") { fileType = AppleS3FileType.StreamsTracks; } else if (type2 == "summary") { fileType = AppleS3FileType.StreamsSummary; } else if (type2 == "containers") { fileType = AppleS3FileType.StreamsContainers; } else if (type2 == "container") { fileType = AppleS3FileType.StreamsContainerTracks; } } else if (type == "demographics") { fileType = AppleS3FileType.Demographics; } var distributorId = _distributerMapping[distributorName]; return new AppleMusicAggregatedS3File() { Date = DateTime.Parse(date), DistributorId = distributorId, Country = country, VendorId = vendorId, Version = version, FilePath = filename, FileType = fileType }; } catch (Exception e) { return null; } } private readonly Dictionary _distributerMapping = new Dictionary() { { "sony", (int)SpotifyAnalyticsAccount.Sony }, { "theorchard", (int)SpotifyAnalyticsAccount.Orchard }, { "smej", (int)SpotifyAnalyticsAccount.SonyMusicEntertainmentJapan }, { "smejintl", (int)SpotifyAnalyticsAccount.SonyMusicEntertainmentJapanInternational }, }; } }