using System; using System.Collections.Generic; using System.IO; using System.Linq; using System.Net; using System.Threading.Tasks; using Amazon.Runtime; using Amazon.S3; using Amazon.S3.Model; using Amazon.S3.Transfer; namespace Sony.Filtr.SpotifyAnalytics { public class SpotifyAnalyticsApi { private readonly AmazonS3Client _s3Client; const string _Bucket = "filtr-spotify-analytics/"; public SpotifyAnalyticsApi(AmazonS3Client s3Client) { _s3Client = s3Client; } public async Task CheckStreamsFileExistsAsync(DateTime analyticsDate, string market, string customerId) { var bucket = GetBucketPath(market, customerId); var fileKey = $"{analyticsDate.ToString("yyyy-MM-dd")}.gz"; return await CheckS3FileExistsAsync(bucket, fileKey); } public async Task> GetStreamDatesAsync(string market, string customerId) { var prefix = GetKey(market, customerId); var objects = await _s3Client.ListObjectsAsync(new ListObjectsRequest() { BucketName = _Bucket, Prefix = prefix, }); List s3Objects = objects.S3Objects; var marker = objects.NextMarker; while (!string.IsNullOrWhiteSpace(marker)) { var moreResults = await _s3Client.ListObjectsAsync(new ListObjectsRequest() { BucketName = _Bucket, Prefix = prefix, Marker = objects.NextMarker }); marker = moreResults.NextMarker; s3Objects.AddRange(moreResults.S3Objects); } List dataFiles = new List(); foreach(var obj in s3Objects.Where(p => !p.Key.EndsWith("/"))) { var date = TryParseDate(obj.Key); if (date.HasValue) { dataFiles.Add(new SpotifyAnalyticsStreamDataFile() { FilePath = obj.Key, Date = date.Value, }); } } return dataFiles; } private DateTime? TryParseDate(string argKey) { var filename = Path.GetFileName(argKey); if (string.IsNullOrWhiteSpace(filename)) return null; var dateFilenamePart = filename.Replace(".gz", string.Empty); DateTime date; if (DateTime.TryParse(dateFilenamePart, out date)) { return date; } return null; } public async Task DownloadStreamsDataFileAsync(DateTime analyticsDate, string market, string customerId, string destinationFilepath) { var bucket = GetBucketPath(market, customerId); var fileKey = $"{analyticsDate.ToString("yyyy-MM-dd")}.gz"; try { await DownloadToFileAsync(destinationFilepath, bucket, fileKey); } catch(AmazonServiceException ex) { if (ex.ErrorCode == "NoSuchKey") return null; throw; } return destinationFilepath; } private static string GetBucketPath(string market, string customerId) { var key = GetKey(market, customerId); return _Bucket + key; } private static string GetKey(string market, string customerId) { string key; if (!string.IsNullOrWhiteSpace(customerId)) { key = $"{customerId}/streams/{market.ToUpperInvariant()}"; } else { key = $"streams/{market.ToUpperInvariant()}"; } return key; } public async Task DownloadTracksDataFileAsync(DateTime analyticsDate, string customerId, string destinationFilepath) { string bucket; if (!string.IsNullOrWhiteSpace(customerId)) { bucket = $"filtr-spotify-analytics/{customerId}/tracks"; } else { bucket = $"filtr-spotify-analytics/tracks"; } var fileKey = $"{analyticsDate.ToString("yyyy-MM-dd")}.gz"; try { await DownloadToFileAsync(destinationFilepath, bucket, fileKey); } catch (AmazonServiceException ex) { if (ex.ErrorCode == "NoSuchKey") return null; throw; } return destinationFilepath; } //public async Task DownloadUsersDataFile(DateTime analyticsDate, string destinationFilepath) //{ // var bucket = $"filtr-spotify-analytics/users"; // var fileKey = $"{analyticsDate.ToString("yyyy-MM-dd")}.gz"; // try // { // await DownloadToFile(destinationFilepath, bucket, fileKey); // } // catch (AmazonServiceException ex) // { // if (ex.ErrorCode == "NoSuchKey") // return null; // throw; // } // return destinationFilepath; //} private async Task DownloadToFileAsync(string destinationFilepath, string bucket, string fileKey) { using(var downloadStream = await DownloadS3FileAsync(bucket, fileKey)) { using(var fileStream = File.Open(destinationFilepath, FileMode.OpenOrCreate)) { await downloadStream.CopyToAsync(fileStream); } } } private async Task CheckS3FileExistsAsync(string bucket, string key) { try { var objectMetadata = await _s3Client.GetObjectMetadataAsync(new GetObjectMetadataRequest() { BucketName = bucket, Key = key }); return objectMetadata.HttpStatusCode == HttpStatusCode.OK; } catch(AmazonServiceException e) { if (e.StatusCode == HttpStatusCode.NotFound) return false; } return false; } private async Task DownloadS3FileAsync(string bucket, string key) { var transferUtility = new TransferUtility(_s3Client); var contentStream = await transferUtility.OpenStreamAsync(new TransferUtilityOpenStreamRequest() { BucketName = bucket, Key = key }); return contentStream; } } public class SpotifyAnalyticsStreamDataFile { public DateTime Date { get; set; } public string FilePath { get; set; } } }