using System; using System.Collections.Generic; using System.IO; using System.IO.Compression; using System.Linq; using System.Threading.Tasks; using CsvHelper.Configuration; namespace Sony.Filtr.AppleMusic.Streams { public class AppleMusicStreamsAnalyzer { private readonly CsvConfiguration _csvConfiguration; public AppleMusicStreamsAnalyzer() { _csvConfiguration = new CsvConfiguration() { HasHeaderRecord = true, Delimiter = "\t", WillThrowOnMissingField = false, }; _csvConfiguration.RegisterClassMap(); _csvConfiguration.RegisterClassMap(); } public async Task> AnalyzeStreamFileAsync(string streamFilePath) { Dictionary streamData = new Dictionary(); using(var gzipFileStream = File.OpenRead(streamFilePath)) { using(var decompressedStream = new GZipStream(gzipFileStream, CompressionMode.Decompress)) { using(var streamReader = new StreamReader(decompressedStream)) { using(var csvReader = new CsvHelper.CsvReader(streamReader, _csvConfiguration)) { Console.WriteLine($"Begin reading csv data from {streamFilePath}"); var rows = csvReader.GetRecords(); Console.WriteLine($"Done reading csv data from {streamFilePath}"); long csvRow = 0; Console.WriteLine($"Begin calculating numbers from {streamFilePath}"); foreach(var row in rows) { if (csvRow % 10000 == 0) { WritePositionToConsole(csvRow); } csvRow++; var storefront = row.Storefront; if (!streamData.ContainsKey(storefront)) { streamData.Add(storefront, new AppleMusicStreamAnalysis() { Storefront = storefront }); } var summary = streamData[storefront].StorefrontSummary; var uniqueUsers = streamData[storefront].UniqueUsers; summary.TotalStreams++; var userId = row.AnonymizedPersonID; uniqueUsers.AllUsers.Add(userId); if (row.SourceOfStream == 0) { summary.SourceOtherStreams++; uniqueUsers.SourceOtherUsers.Add(userId); } else if (row.SourceOfStream == 1) { summary.SourceLibraryStreams++; uniqueUsers.SourceLibraryUsers.Add(userId); } else if (row.SourceOfStream == 2) { summary.SourceSearchStreams++; uniqueUsers.SourceSearchUsers.Add(userId); } else if (row.SourceOfStream == 3) { summary.SourceDiscoveryStreams++; uniqueUsers.SourceDiscoveryUsers.Add(userId); } else if (row.SourceOfStream == 4) { summary.SourceMusicKitStreams++; uniqueUsers.SourceDiscoveryUsers.Add(userId); } if (row.ContainerType == 0) { summary.ContainerNoneStreams++; uniqueUsers.ContainerNoneUsers.Add(userId); } else if (row.ContainerType == 1) { summary.ContainerRadioStreams++; uniqueUsers.ContainerRadioUsers.Add(userId); } else if (row.ContainerType == 2) { summary.ContainerPlaylistStreams++; uniqueUsers.ContainerPlaylistUsers.Add(userId); if (IsOtherPlaylistType(row.ContainerSubType)) { uniqueUsers.ContainerOtherPlaylistsUsers.Add(userId); } IncrementContainerSubTypeCount(row, summary); var playlistId = row.ContainerID; if (!string.IsNullOrWhiteSpace(row.ContainerID)) { if (!streamData[storefront].Playlist.ContainsKey(playlistId)) { streamData[storefront].Playlist.Add(playlistId, new AppleMusicPlaylistAnalysis()); } streamData[storefront].Playlist[playlistId].TotalStreams++; streamData[storefront].Playlist[playlistId].Users.Add(row.AnonymizedPersonID); } } else if (row.ContainerType == 3) { summary.ContainerAlbumStreams++; uniqueUsers.ContainerAlbumUsers.Add(userId); } } Console.WriteLine($"Done calculating numbers from {streamFilePath}"); Console.WriteLine($"Done reading {streamFilePath}"); } } } } return streamData.Values.ToList(); } readonly List _otherPlaylistsContainerSubTypes = new List() { 2, 3, 4 }; private bool IsOtherPlaylistType(int? rowContainerSubType) { return rowContainerSubType != null && _otherPlaylistsContainerSubTypes.Contains(rowContainerSubType.Value); } private static void IncrementContainerSubTypeCount(AppleMusicStream row, AppleMusicStorefrontAnalysis summary) { switch (row.ContainerSubType) { case 1: summary.ContainerSubTypePrivateUserPlaylist++; break; case 2: summary.ContainerSubTypeEditorialPlaylist++; break; case 3: summary.ContainerSubTypeArtistPlaylist++; break; case 4: summary.ContainerSubTypeCuratorPlaylist++; break; case 5: summary.ContainerSubTypeSeeded++; break; case 6: summary.ContainerSubTypeFormatStation++; break; case 7: summary.ContainerSubTypeEditorialStation++; break; case 8: summary.ContainerSubTypePersonalMixPlaylist++; break; default: summary.ContainerSubTypeNonApplicable++; break; } } private static readonly object _ConsoleWriterLock = new object(); private void WritePositionToConsole(long lineNumber) { lock (_ConsoleWriterLock) { ClearCurrentConsoleLine(); Console.SetCursorPosition(0, Console.CursorTop); Console.Write(lineNumber); } } private static void ClearCurrentConsoleLine() { var currentLineCursor = Console.CursorTop; Console.SetCursorPosition(0, Console.CursorTop); Console.Write(new string(' ', Console.WindowWidth)); Console.SetCursorPosition(0, currentLineCursor); } } public sealed class AppleMusicContentMap : CsvClassMap { public AppleMusicContentMap() { Map(m => m.AppleIdentifier).Index(0); Map(m => m.Isrc).Index(1); //Map(m => m.Title).Index(2); //Map(m => m.Artist).Index(3); //Map(m => m.ItemType).Index(4); //Map(m => m.MediaType).Index(5); //Map(m => m.VendorIdentifier).Index(6); //Map(m => m.LabelStudioNetwork).Index(7); //Map(m => m.Grid).Index(8); } } public sealed class AppleMusicStreamMap : CsvClassMap { public AppleMusicStreamMap() { Map(m => m.Datestamp).Index(0); Map(m => m.IngestDatestamp).Index(1); Map(m => m.Identifier).Index(2); Map(m => m.Storefront).Index(3); Map(m => m.AnonymizedPersonID).Index(4); //Map(m => m.MembershipType).Index(5); //Map(m => m.MembershipMode).Index(6); //Map(m => m.PostalCode).Index(7); //Map(m => m.DeviceType).Index(8); //Map(m => m.OperatingSystem).Index(9); //Map(m => m.UTCOffset).Index(10); //Map(m => m.ActionType).Index(11); //Map(m => m.EndReasonType).Index(12); //Map(m => m.Offline).Index(13); Map(m => m.SourceOfStream).Index(14); Map(m => m.ContainerType).Index(15); //Map(m => m.StreamTimestamp).Index(16); //Map(m => m.StreamStartPosition).Index(17); //Map(m => m.StreamDuration).Index(18); //Map(m => m.MediaDuration).Index(19); Map(m => m.ContainerSubType).Index(20); Map(m => m.ContainerID).Index(21); //Map(m => m.ContainerName).Index(22); } } public class AppleMusicContent { public long AppleIdentifier { get; set; } public string Isrc { get; set; } //public string Title { get; set; } //public string Artist { get; set; } //public int ItemType { get; set; } //public int MediaType { get; set; } //public string VendorIdentifier { get; set; } //public string LabelStudioNetwork { get; set; } //public string Grid { get; set; } } public class AppleMusicStream { public string Datestamp { get; set; } public string IngestDatestamp { get; set; } public long Identifier { get; set; } public string Storefront { get; set; } public string AnonymizedPersonID { get; set; } //public string MembershipType { get; set; } //public string MembershipMode { get; set; } //public string PostalCode { get; set; } //public string DeviceType { get; set; } //public string OperatingSystem { get; set; } //public string UTCOffset { get; set; } //public string ActionType { get; set; } //public string EndReasonType { get; set; } //public string Offline { get; set; } public int SourceOfStream { get; set; } public int ContainerType { get; set; } //public string StreamTimestamp { get; set; } //public string StreamStartPosition { get; set; } //public string StreamDuration { get; set; } //public string MediaDuration { get; set; } public int? ContainerSubType { get; set; } public string ContainerID { get; set; } //public string ContainerName { get; set; } } public class AppleMusicStreamAnalysis { public AppleMusicStreamAnalysis() { StorefrontSummary = new AppleMusicStorefrontAnalysis(); Playlist = new Dictionary(); UniqueUsers = new AppleMusicAnalyticsUniqueUsers(); } public string Storefront { get; set; } public AppleMusicStorefrontAnalysis StorefrontSummary { get; set; } public AppleMusicAnalyticsUniqueUsers UniqueUsers { get; set; } public Dictionary Playlist { get; set; } } public class AppleMusicStorefrontAnalysis { public long TotalStreams { get; set; } public long SourceSearchStreams { get; set; } public long SourceOtherStreams { get; set; } public long SourceDiscoveryStreams { get; set; } public long SourceMusicKitStreams { get; set; } public long SourceLibraryStreams { get; set; } public long ContainerPlaylistStreams { get; set; } public long ContainerRadioStreams { get; set; } public long ContainerAlbumStreams { get; set; } public long ContainerNoneStreams { get; set; } public long ContainerSubTypeNonApplicable { get; set; } public long ContainerSubTypePrivateUserPlaylist { get; set; } public long ContainerSubTypeEditorialPlaylist { get; set; } public long ContainerSubTypeArtistPlaylist { get; set; } public long ContainerSubTypeCuratorPlaylist { get; set; } public long ContainerSubTypeSeeded { get; set; } public long ContainerSubTypeFormatStation { get; set; } public long ContainerSubTypeEditorialStation { get; set; } public long ContainerSubTypePersonalMixPlaylist { get; set; } } public class AppleMusicAnalyticsUniqueUsers { public AppleMusicAnalyticsUniqueUsers() { AllUsers = new HashSet(); SourceSearchUsers = new HashSet(); SourceOtherUsers = new HashSet(); SourceDiscoveryUsers = new HashSet(); SourceLibraryUsers = new HashSet(); SourceMusicKitUsers = new HashSet(); ContainerPlaylistUsers = new HashSet(); ContainerRadioUsers = new HashSet(); ContainerAlbumUsers = new HashSet(); ContainerNoneUsers = new HashSet(); ContainerOtherPlaylistsUsers = new HashSet(); } public HashSet AllUsers { get; set; } public HashSet SourceSearchUsers { get; set; } public HashSet SourceOtherUsers { get; set; } public HashSet SourceDiscoveryUsers { get; set; } public HashSet SourceMusicKitUsers { get; set; } public HashSet SourceLibraryUsers { get; set; } public HashSet ContainerPlaylistUsers { get; set; } public HashSet ContainerOtherPlaylistsUsers { get; set; } public HashSet ContainerRadioUsers { get; set; } public HashSet ContainerAlbumUsers { get; set; } public HashSet ContainerNoneUsers { get; set; } } public class AppleMusicPlaylistAnalysis { public AppleMusicPlaylistAnalysis() { Users = new HashSet(); } public long TotalStreams { get; set; } public HashSet Users { get; set; } } }