using System; using System.Collections.Generic; using System.Globalization; using System.IO; using System.IO.Compression; using System.Linq; using System.Threading.Tasks; using Amazon.S3; using Amazon.S3.Model; using Amazon.S3.Transfer; using Sony.Filtr.Contracts.Definitions; using Sony.Filtr.SpotifyAnalytics.Models; namespace Sony.Filtr.SpotifyAnalytics { public class SpotifyStreamingAggregatedReportApi { private readonly AmazonS3Client _s3Client; private readonly string _Bucket; public SpotifyStreamingAggregatedReportApi(AmazonS3Client s3Client, string bucket) { _s3Client = s3Client; this._Bucket = bucket; } public async Task> GetFilesAsync(DateTime date) { List filenames = new List(); var dateString = date.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture); var streamObjects = await _s3Client.ListObjectsAsync(new ListObjectsRequest() { BucketName = _Bucket, Prefix = $"spotify/{dateString}/", }); filenames.AddRange(streamObjects.S3Objects.Select(p=> p.Key).ToList()); var nextMarker = streamObjects.NextMarker; while (!string.IsNullOrWhiteSpace(nextMarker)) { var moreResults = await _s3Client.ListObjectsAsync(new ListObjectsRequest() { BucketName = _Bucket, Prefix = $"spotify/{dateString}/", Marker = nextMarker }); nextMarker = moreResults.NextMarker; filenames.AddRange(moreResults.S3Objects.Select(p => p.Key).ToList()); } var fileNames = filenames.Select(p => ParseSpotifyS3FilePath(p)).Where(p => p != null).ToList(); return fileNames; } public async Task DownloadToFileAsync(string destinationFilepath, string fileKey) { using (var downloadStream = await DownloadS3FileAsync(_Bucket, fileKey)) { using (var fileStream = File.Open(destinationFilepath, FileMode.OpenOrCreate)) { using(var gunzippedStream = new GZipStream(downloadStream, CompressionMode.Decompress)) { await gunzippedStream.CopyToAsync(fileStream); } } } } private async Task DownloadS3FileAsync(string bucket, string key) { var transferUtility = new TransferUtility(_s3Client); var contentStream = await transferUtility.OpenStreamAsync(new TransferUtilityOpenStreamRequest() { BucketName = bucket, Key = key, }); return contentStream; } private SpotifyS3File ParseSpotifyS3FilePath(string filename) { //spotify_2017-11-21_sony_v1_af_streams_playlist_tracks.ndjson try { var parsedFilename = Path.GetFileNameWithoutExtension(filename); if (string.IsNullOrWhiteSpace(parsedFilename)) return null; var fileParts = parsedFilename.Split('_'); if (!fileParts.Any()) return null; if (fileParts.ElementAt(0) != "spotify") return null; var date = fileParts.ElementAt(1); var distributorName = fileParts.ElementAt(2); var version = ParseVersion(fileParts); string country = "global"; SpotifyS3FileType fileType; var countryOrType = fileParts.ElementAt(4); if (countryOrType == "saves" || countryOrType == "demographics") { fileType = GetFileType(countryOrType); } else if (countryOrType == "listenersummary") { var listenerSummaryPart = fileParts.ElementAt(5); if (listenerSummaryPart == "playlist") { fileType = SpotifyS3FileType.ListenerSummaryPlaylistTracks; } else if (listenerSummaryPart == "playlists") { fileType = SpotifyS3FileType.ListenerSummaryPlaylists; } else if (listenerSummaryPart == "tracks") { fileType = SpotifyS3FileType.ListenerSummaryTracks; } else { return null; } } else { country = countryOrType; fileType = GetFileType(fileParts.ElementAt(6)); } var distributorId = _distributerMapping[distributorName]; return new SpotifyS3File() { Date = DateTime.Parse(date), DistributorId = distributorId, Country = country, Version = version, FilePath = filename, FileType = fileType }; } catch(Exception e) { return null; } } private static int ParseVersion(string[] fileParts) { var versionString = fileParts.ElementAt(3); if (versionString.Equals("v1")) return 1; if (versionString.Equals("v2")) return 2; throw new NotSupportedException("Unknown version in filename"); } private SpotifyS3FileType GetFileType(string type) { if (type.Equals("playlists")) return SpotifyS3FileType.Playlists; if (type.Equals("playlist")) return SpotifyS3FileType.PlaylistTracks; if (type.Equals("tracks")) return SpotifyS3FileType.Tracks; if (type.Equals("summary")) return SpotifyS3FileType.Summary; if (type.Equals("demographics")) return SpotifyS3FileType.Demographics; if (type.Equals("saves")) return SpotifyS3FileType.SaveSkips; return SpotifyS3FileType.Unknown; } readonly Dictionary _distributerMapping = new Dictionary() { { "sony", (int)SpotifyAnalyticsAccount.Sony }, { "theorchard", (int)SpotifyAnalyticsAccount.Orchard }, { "smej", (int)SpotifyAnalyticsAccount.SonyMusicEntertainmentJapan }, { "smejintl", (int)SpotifyAnalyticsAccount.SonyMusicEntertainmentJapanInternational }, }; } }