using System; using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; using AngleSharp; using AngleSharp.Dom; using AngleSharp.Dom.Html; using AngleSharp.Extensions; using MoreLinq; using NLog; using PetaPoco.Business; using Sony.Filtr.Contracts.Entities; using Sony.Filtr.Utility.Extensions; namespace Sony.Filtr.Tasks.Tasks.EveryNoise { public class EveryNoiseReleaseImportTask : IScheduledTask { private Logger _logger; public EveryNoiseReleaseImportTask() { _logger = LogManager.GetLogger("EveryNoiseReleaseImportTask"); } public async Task ExecuteAsync(Guid scheduledTaskLogId) { var config = Configuration.Default.WithDefaultLoader(); var browsingContext = AngleSharp.BrowsingContext.New(config); var listingUrl = "http://everynoise.com/sorting_hat_closet/"; _logger.Info($"Doing inital request to listing url: {listingUrl}"); var listingPageDoc = await browsingContext.OpenAsync(listingUrl); var datePages = listingPageDoc.Links.OfType().Select(p => new { Date = GetDate(p.Text()), LinkUrl = p.Href }).Where(p => p.Date.HasValue && !string.IsNullOrWhiteSpace(p.LinkUrl)).ToList(); _logger.Info($"Listing page has {datePages.Count} entries."); var fetchedReleaseDates = PetaPocoRepository.Instance.Fetch(); var fetchedReleaseDatesDic = fetchedReleaseDates.Select(p=> p.Date).Distinct().ToHashSet(); _logger.Info($"We have {fetchedReleaseDatesDic.Count()} days in database already."); var datePagesToImport = datePages.Where(p => !fetchedReleaseDatesDic.Contains(p.Date.Value)).ToList(); _logger.Info($"There are {datePagesToImport.Count()} days to import."); await datePagesToImport.ForEachAsync(2, async datePage => { _logger.Info($"Fetching {datePage.LinkUrl} for date {datePage.Date}."); var releasesPageData = await GetReleasePageDataAsync(browsingContext, datePage.Date, datePage.LinkUrl); var releases = releasesPageData.Releases; var uniqueReleases = releases.DistinctBy(p => p.AlbumUri).ToList(); _logger.Info($"{datePage.LinkUrl} for date {datePage.Date} contains {uniqueReleases.Count} releases."); await PetaPocoRepository.Instance.BulkInsertAsync(uniqueReleases, ignoreExisting: true, batchSize: 100); PetaPocoRepository.Instance.Insert(new EveryNoiseReleaseDate() { Date = datePage.Date.Value, Url = datePage.LinkUrl }); try { await SaveGenreClusterAsync(datePage.Date.Value, releasesPageData.Clusters); } catch(Exception ex) { _logger.Error(ex); } _logger.Info($"Done importing {datePage.LinkUrl} for date {datePage.Date}."); }); return null; } private async Task SaveGenreClusterAsync(DateTime date, List clusters) { await clusters.ItemIndex().ForEachAsync(2, async genreCluster => { var addedCluster = new EveryNoiseCluster() { Date = date, Order = genreCluster.Index, }; PetaPocoRepository.Instance.Insert(addedCluster); var dbGenres = genreCluster.Item.Genres.Select(p => new EveryNoiseClusterGenre() { ClusterId = addedCluster.Id, Genre = p }).ToList(); await PetaPocoRepository.Instance.BulkInsertAsync(dbGenres); var dbGenreReleases = genreCluster.Item.ReleaseAlbumId.Select(p => new EveryNoiseClusterRelease() { ClusterId = addedCluster.Id, AlbumId = p }).ToList(); await PetaPocoRepository.Instance.BulkInsertAsync(dbGenreReleases); }); } private async Task GetReleasePageDataAsync(IBrowsingContext browsingContext, DateTime? date, string url) { List releases = new List(); var doc = await browsingContext.OpenAsync(url); var allReleasesElement = doc.GetElementsByClassName("allreleases").FirstOrDefault(); if (allReleasesElement != null) { var releaseElements = allReleasesElement.Children.Where(p => p.ClassName != "subtitle").ToList(); //Filter heading foreach(var releaseElement in releaseElements) { var release = new EveryNoiseRelease { Date = date, ArtistName = GetArtistName(releaseElement), Title = GetTitle(releaseElement), ArtistRank = GetArtistRank(releaseElement), ReleaseType = GetReleaseType(releaseElement), AlbumUri = GetAlbumLink(releaseElement)?.Uri, TrackCount = GetTrackCount(releaseElement) }; releases.Add(release); } } var leftGenresClusters = doc.GetElementsByClassName("leftgenre genre"); List genreClusters = new List(); foreach(var cluster in leftGenresClusters) { var genreElements = cluster.QuerySelectorAll("a.clustergenre"); var genres = genreElements.Select(p => p.Text()).ToList(); var albumElements = cluster.QuerySelectorAll("div.albumwrapper"); var albumIds = albumElements.Select(p => p.GetAttribute("albumid")).Where(p=> !string.IsNullOrWhiteSpace(p)).Select(p=> new SpotifyLink(p).ExtractAlbumID()).ToList(); genreClusters.Add(new GenreCluster() { Genres = genres, ReleaseAlbumId = albumIds, }); } return new ReleasePageData() { Releases = releases, Clusters = genreClusters }; } private string GetArtistName(IElement releaseElement) { var artistElement = releaseElement.Descendents().FirstOrDefault(p => p.ClassList.Contains("creditartist")); return artistElement?.Text(); } private static string GetTitle(IElement releaseElement) { var releaseNameElement = releaseElement.Descendents().FirstOrDefault(p => p.ClassList.Contains("creditrelease")); var title = releaseNameElement?.Text(); if (title != null && title.StartsWith("\"") && title.Length > 2) { title = title.Remove(0, 1); } if (title != null && title.EndsWith("\"") && title.Length > 2) { title = title.Remove(title.Length - 1, 1); } return title; } private int? GetTrackCount(IElement releaseElement) { var trackCountElement = releaseElement.Descendents().FirstOrDefault(p => p.ClassList.Contains("trackcount")); var trackCountText = trackCountElement.Text(); if (int.TryParse(trackCountText, out int trackCount)) { return trackCount; } return null; } private SpotifyLink GetAlbumLink(IElement releaseElement) { var playElement = releaseElement.Descendents().FirstOrDefault(p => p.ClassList.Contains("play")); if (playElement != null) { var albumUri = playElement.GetAttribute("albumId"); if (!string.IsNullOrWhiteSpace(albumUri)) { return new SpotifyLink(albumUri); } } return null; } private int? GetArtistRank(IElement releaseElement) { var title = releaseElement.GetAttribute("title"); title = title.Replace("artist rank: ", string.Empty); if (int.TryParse(title, out int artistRank)) { return artistRank; } return null; } private EveryNoiseReleaseType? GetReleaseType(IElement releaseElement) { if (releaseElement.ClassList.Contains("single")) return EveryNoiseReleaseType.Single; if (releaseElement.ClassList.Contains("album")) return EveryNoiseReleaseType.Album; if (releaseElement.ClassList.Contains("other")) return EveryNoiseReleaseType.Other; return null; } private DateTime? GetDate(string text) { if (DateTime.TryParse(text, out DateTime date)) { return date; } return null; } } }