using Amazon; using Amazon.S3; using Amazon.S3.Transfer; using Sony.Filtr.Contracts.Entities; using System; using System.IO; using System.IO.Compression; using System.Text; using System.Threading; using System.Threading.Tasks; using System.Threading.Tasks.Dataflow; namespace Sony.Filtr.Core.ResponseCollector { public class ResponseCollector { class PrepareFileData { public readonly string Url; public readonly string Content; public readonly string JobName; public PrepareFileData(string url, string content, string jobName) { this.Url = url; this.Content = content; this.JobName = jobName; } } private readonly S3BucketReferenceWithRegion _destinationBucket; private ActionBlock AppendResponseBlock; public ResponseCollector(S3BucketReferenceWithRegion destinationBucket) { _destinationBucket = destinationBucket; this.AppendResponseBlock = new ActionBlock(PrepareFile, new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 1 }); } public void UploadFile(string filePath, string folderName) { this.AppendResponseBlock.Complete(); this.AppendResponseBlock.Completion.Wait(); if (File.Exists(filePath)) { var s3Client = new AmazonS3Client(RegionEndpoint.GetBySystemName(_destinationBucket.Region)); var transferUtility = new TransferUtility(s3Client); var compressedFilename = CompressFile(filePath); File.Delete(filePath); var formattedFilePath = compressedFilename.Replace("\\", "/"); transferUtility.Upload(new TransferUtilityUploadRequest() { BucketName = _destinationBucket.Bucket, FilePath = formattedFilePath, Key = folderName + "/" + DateTime.UtcNow.ToString("MM-dd-yyyy-HH-mm-ss") + ".json.gz", CannedACL = S3CannedACL.BucketOwnerFullControl, AutoCloseStream = true }); File.Delete(compressedFilename); } } public async Task PrepareFile(string url, string jobName, string response) { this.AppendResponseBlock.Post(new PrepareFileData(url, response, jobName)); } private void PrepareFile(PrepareFileData data) { if (!string.IsNullOrEmpty(data.Content)) { var path = GetOrCreateTempDirectory(); var filename = @"ChartsResponse.json"; var filePath = Path.Combine(path, filename); FileInfo file = new FileInfo(filePath); using (var fileStream = file.OpenWrite()) { fileStream.Write(Encoding.UTF8.GetPreamble(), 0, Encoding.UTF8.GetPreamble().Length); using (var fileWriter = new StreamWriter(fileStream)) { var pattern = "Date: {0}, Job: {1}, URL: {2}, Response: {3}" + Environment.NewLine; string formattedResponse = String.Format(pattern, DateTime.UtcNow, data.JobName, data.Url, data.Content); fileWriter.Write(formattedResponse + Environment.NewLine); } } } } private string GetOrCreateTempDirectory() { var dirPath = "ResponseData"; if (Directory.Exists(dirPath)) { return dirPath; } Directory.CreateDirectory(dirPath); return dirPath; } private string CompressFile(string filepath) { var originalFilename = Path.GetFileName(filepath); var compressedFilename = $"{originalFilename}.gz"; var directory = GetOrCreateTempDirectory(); var compressedFilePath = Path.Combine(directory, compressedFilename); using (FileStream originalFileStream = File.OpenRead(filepath)) { using (FileStream destinationCompressedFile = File.OpenWrite(compressedFilePath)) { using (GZipStream compressionStream = new GZipStream(destinationCompressedFile, CompressionMode.Compress)) { originalFileStream.CopyTo(compressionStream); } } } return compressedFilePath; } } }