using System; using System.Collections.Concurrent; using System.Collections.Generic; using System.IO; using System.Linq; using System.Threading.Tasks; using CsvHelper.Configuration; using DocumentFormat.OpenXml.Spreadsheet; using NLog; using Sony.Filtr.Contracts.Definitions; using Sony.Filtr.Contracts.Entities; using Sony.Filtr.Utility.Extensions; using SpreadsheetLight; namespace Sony.Filtr.Tasks.Tasks.SonyReleases { public class SonyReleaseParser { private readonly Logger _logger; public SonyReleaseParser() { _logger = LogManager.GetLogger("ImportNewSonyReleasesTask"); } public async Task> ParseSonyAlbumUpdateFileAsync(string filename, Stream filecontent) { ConcurrentBag albums = new ConcurrentBag(); _logger.Debug($"Begin loading file {filename}"); var spreadsheet = new SLDocument(filecontent); _logger.Debug($"Done loading file {filename}"); var sharedStrings = spreadsheet.GetSharedStrings(); var sharedStringsDic = new ConcurrentDictionary(sharedStrings.ItemIndex().ToDictionary(k => k.Index, v => v.Item.GetText())); _logger.Debug($"Done reading shared strings from file {filename}"); var cells = spreadsheet.GetCells().Skip(1).ToList(); // Skip header row. _logger.Debug($"Done reading cells from file {filename}"); var cellCount = cells.Count; foreach (var cell in cells.ItemIndex()) { _logger.Debug($"{filename}: {cell.Index} / {cellCount}"); try { var realCell = cell.Item.Value; var upcCell = realCell.GetValueOrDefault(1); var countryCell = realCell.GetValueOrDefault(2); var partnerCell = realCell.GetValueOrDefault(5); var labelCell = realCell.GetValueOrDefault(6); var actionTypeCell = realCell.GetValueOrDefault(8); var upcValue = upcCell.DataType == CellValues.SharedString ? GetValue(upcCell, sharedStringsDic) : (upcCell.DataType == CellValues.Number ? upcCell.NumericValue.ToString() : null); var countryValue = GetValue(countryCell, sharedStringsDic); var partnerValue = GetValue(partnerCell, sharedStringsDic); var labelValue = GetValue(labelCell, sharedStringsDic); bool revoked = false; if (actionTypeCell != null) { var actionTypeValue = GetValue(actionTypeCell, sharedStringsDic); revoked = actionTypeValue != null && actionTypeValue.Equals("rev", StringComparison.InvariantCultureIgnoreCase); } if (partnerValue != null) { var musicService = ParseMusicService(partnerValue); if (!string.IsNullOrWhiteSpace(upcValue) && musicService.HasValue) { var regions = countryValue.Split(new[] { "," }, StringSplitOptions.RemoveEmptyEntries).Select(s => s.Trim()).ToList(); var album = new SonyUpcAlbum { UPC = upcValue, Label = labelValue, Regions = regions.Select(s => new SonyUpcRegion { Region = s, MusicService = musicService.Value, ReleaseType = SpotifyAnalyticsAccount.Sony, }).ToList(), MusicService = musicService.Value, Revoked = revoked, }; albums.Add(album); } else { _logger.Error($"Invalid upc or music service(partner) in {filename} at index {cell.Index}"); } } } catch (Exception ex) { _logger.Error($"Error when importing file {filename} at index {cell.Index}"); } } return albums.ToList(); } public async Task> ParseSonyAlbumsAsync(string file, MusicService musicService) { ConcurrentBag albums = new ConcurrentBag(); _logger.Debug($"Begin loading file {file}"); var spreadsheet = new SpreadsheetLight.SLDocument(file); _logger.Debug($"Done loading file {file}"); var sharedStrings = spreadsheet.GetSharedStrings(); _logger.Debug($"Done reading shared strings from file {file}"); var cells = spreadsheet.GetCells().Skip(1).ToList(); // Skip header row. _logger.Debug($"Done reading cells from file {file}"); var cellCount = cells.Count; foreach (var cell in cells.ItemIndex()) { _logger.Debug($"{file}: {cell.Index} / {cellCount}"); try { var realCell = cell.Item.Value; var upcCell = realCell.ElementAt(0); var countryCell = realCell.ElementAt(1); var labelCell = realCell.ElementAt(5); var upcValue = GetValue(upcCell.Value, sharedStrings); var countryValue = GetValue(countryCell.Value, sharedStrings); var labelValue = GetValue(labelCell.Value, sharedStrings); if (!string.IsNullOrWhiteSpace(upcValue)) { var regions = countryValue.Split(new[] { "," }, StringSplitOptions.RemoveEmptyEntries).Select(s => s.Trim()).ToList(); var album = new SonyUpcAlbum() { UPC = upcValue, Label = labelValue, Regions = regions.Select(s => new SonyUpcRegion() { Region = s, MusicService = musicService, ReleaseType = SpotifyAnalyticsAccount.Sony }).ToList() }; albums.Add(album); } } catch (Exception ex) { _logger.Error($"Error when importing file {file} at index {cell.Index}"); } } return albums.ToList(); } public List ParseIsrcReleaseFile(Stream fileContent) { var spreadsheet = new SLDocument(fileContent); var sharedStrings = spreadsheet.GetSharedStrings(); List releases = new List(); foreach (var cell in spreadsheet.GetCells()) { var isrcCell = cell.Value.ElementAt(0); var countryCell = cell.Value.ElementAt(1); var isrcValue = GetValue(isrcCell.Value, sharedStrings); var countryValue = GetValue(countryCell.Value, sharedStrings); if (isrcValue != null) { releases.Add(new SonyRelease() { Isrc = isrcValue, Regions = countryValue.Split(new[] { "," }, StringSplitOptions.RemoveEmptyEntries).Select(s => s.Trim()).ToList() }); } } return releases; } public List ParseSmejUpcReleaseFile(Stream fileStream) { var csvConfiguration = new CsvConfiguration() { HasHeaderRecord = true, Delimiter = "\t", WillThrowOnMissingField = false, }; csvConfiguration.RegisterClassMap(); using(var streamReader = new StreamReader(fileStream)) { using (var csvReader = new CsvHelper.CsvReader(streamReader, csvConfiguration)) { var records = csvReader.GetRecords().ToList(); return records; } } } private string GetValue(SLCell cell, IDictionary sharedStringsDic) { if (cell.DataType == CellValues.SharedString) { return GetValueOrDefault(sharedStringsDic,(int)cell.NumericValue); } return null; } private static TValue GetValueOrDefault(IDictionary dictionary, TKey key) { TValue value; return dictionary.TryGetValue(key, out value) ? value : default(TValue); } private string GetValue(SLCell cell, SLRstType[] sharedStrings) { if (cell.DataType == CellValues.SharedString) { var sharedString = sharedStrings[(int)cell.NumericValue]; var stringValue = sharedString.GetText(); return stringValue; } return null; } private string GetValue(SLCell cell, List sharedStrings) { if (cell.DataType == CellValues.SharedString) { var sharedString = sharedStrings.ElementAt((int)cell.NumericValue); var stringValue = sharedString.GetText(); return stringValue; } return null; } //private string GetValue(SLCell cell, List sharedStrings) //{ // if (cell.DataType == CellValues.SharedString) // { // var sharedString = sharedStrings.ElementAt((int)cell.NumericValue); // var stringValue = sharedString.GetText(); // return stringValue; // } // return null; //} private static MusicService? ParseMusicService(string partnerValue) { if (partnerValue.Equals("Spotify", StringComparison.InvariantCultureIgnoreCase)) return MusicService.Spotify; if (partnerValue.Equals("Deezer", StringComparison.InvariantCultureIgnoreCase)) return MusicService.Deezer; if (partnerValue.Equals("iTunes", StringComparison.InvariantCultureIgnoreCase)) return MusicService.AppleMusic; //Ugly hax... return null; } } sealed class SmejUpcReleaseRowMap : CsvClassMap { public SmejUpcReleaseRowMap() { Map(row => row.Upc).Index(0); Map(row => row.Date).Index(1); Map(row => row.CountryCode).Index(2); } } public class SmejUpcReleaseRow { public string Upc { get; set; } public DateTime Date { get; set; } public string CountryCode { get; set; } } }