using System; using System.Collections.Concurrent; using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; using ConcurrentCollections; using MoreLinq; using NLog; using Sony.Filtr.Utility.Extensions; using Sony.Filtr.YouTube.Api; using Sony.Filtr.YouTube.Channels; using Sony.Filtr.YouTube.Videos; namespace Sony.Filtr.Tasks.Tasks.Youtube { public class ImportYouTubeVideosFromChannelsTask : IScheduledTask { private readonly YouTubeApi _youtubeApi; private readonly YouTubeChannelManager _youTubeChannelManager; private readonly YouTubeVideoManager _youTubeVideosManager; private readonly Logger _logger; public ImportYouTubeVideosFromChannelsTask(YouTubeApi youtubeApi, YouTubeChannelManager youTubeChannelManager, YouTubeVideoManager youTubeVideosManager) { _youtubeApi = youtubeApi; _youTubeChannelManager = youTubeChannelManager; _youTubeVideosManager = youTubeVideosManager; _logger = LogManager.GetLogger("ImportYouTubeVideosFromChannels"); } public async Task ExecuteAsync(Guid scheduledTaskLogId) { _logger.Info("Importing youtube channels and videos and stats"); var allChannels = _youTubeChannelManager.GetChannels(); _logger.Info($"Doing import for {allChannels.Count} channels"); var channelsFromApi = await GetChannelsFromApi(allChannels); _logger.Info($"Saving stats for {channelsFromApi.Count()} channels from YouTube API"); await SaveChannelStats(channelsFromApi); _logger.Info($"Importing videos from channels..."); var savedVideoIds = new ConcurrentHashSet(); await channelsFromApi.ForEachAsync(5, async channel => { try { var channelVideosFromApi = await GetVideosForChannel(channel); await SaveVideosForChannel(channel, channelVideosFromApi); await SaveVideoStats(channelVideosFromApi); channelVideosFromApi.ForEach(v => savedVideoIds.Add(v.id)); _logger.Info($"Saved {channelVideosFromApi.Count()} stats and videos for channel \"{channel.snippet?.title}\""); }catch(Exception ex) { _logger.Error(ex, $"Error when getting videos for channel with ID {channel.id}"); } }); var videos = _youTubeVideosManager.GetVideos(); await ImportVideoStats(videos, savedVideoIds); return null; } private async Task ImportVideoStats(List videos, ConcurrentHashSet savedVideoIds) { var videoIds = videos.Select(v => v.Id).Except(savedVideoIds); var videoBatches = videoIds.Batch(2500); foreach(var batch in videoBatches) { var videosFromApi = await _youtubeApi.GetAllVideosAsync(batch.ToList()); await SaveVideoStats(videosFromApi); } } private async Task SaveVideoStats(IEnumerable videos) { var videoStats = videos.Select(v => new YouTubeVideoStatistics { VideoId = v.id, Date = DateTime.UtcNow.Date, Views = v.statistics.viewCount, Likes = v.statistics.likeCount, Dislikes = v.statistics.dislikeCount, Comments = v.statistics.commentCount, Favorites = v.statistics.favoriteCount }); await _youTubeVideosManager.AddVideoStatisticsAsync(videoStats); } private async Task SaveVideosForChannel(ChannelItem channel, IEnumerable videos) { if (videos.IsNullOrEmpty()) return; var importVideos = videos.Select(v => new YouTubeVideo(v)).ToList(); await _youTubeVideosManager.AddVideosAsync(importVideos); } private async Task> GetVideosForChannel(ChannelItem channel) { var channelSectionsResponse = await _youtubeApi.GetChannelSectionsAsync(channel.id); var musicVideoPlaylistId = channelSectionsResponse?.items?.FirstOrDefault(i => !string.IsNullOrWhiteSpace(i?.snippet?.title) && i.snippet.title.Equals("Music Videos", StringComparison.InvariantCultureIgnoreCase))?.contentDetails?.playlists?.FirstOrDefault(); if (string.IsNullOrWhiteSpace(musicVideoPlaylistId)) { musicVideoPlaylistId = channel.contentDetails?.relatedPlaylists.uploads; } var channelVideos = await _youtubeApi.GetAllPlaylistItemsAsync(musicVideoPlaylistId); var allVideosWithStats = await _youtubeApi.GetAllVideosAsync(channelVideos.Select(v => v.snippet.resourceId.videoId).ToList()); return allVideosWithStats; } private async Task SaveChannelStats(IEnumerable channelsFromApi) { var channelStats = channelsFromApi.Select(c => new YouTubeChannelStatistics { ChannelId = c.id, Date = DateTime.UtcNow.Date, Comments = c.statistics.commentCount, Videos = c.statistics.videoCount, Views = c.statistics.viewCount, Subscribers = c.statistics.subscriberCount, HiddenSubscribers = c.statistics.hiddenSubscriberCount }); await _youTubeChannelManager.AddChannelStatisticsAsync(channelStats); } private async Task> GetChannelsFromApi(IEnumerable allChannels) { var channelsFromApi = new ConcurrentBag(); var batches = allChannels.Select(c => c.Id).Batch(50); await batches.ToList().ForEachAsync(5, async batch => { try { var channelsResponse = await _youtubeApi.GetChannelsAsync(batch.ToList()); channelsResponse?.items.ForEach(channel => channelsFromApi.Add(channel)); }catch(Exception ex) { _logger.Error(ex, "Error in fetching channels"); } }); return channelsFromApi; } } }