"""YouTube scrapping flow.""" import logging import sys from pyyoutube import Api, PyYouTubeException from requests import ReadTimeout from snowflake_executor import YouTubeStatsSFExecutor import config import helpers def get_id_from_link(text): """Return YouTube video link from config.""" return text. \ replace(config.VIDEO_LINK, '') class YouTubeTracker: """YouTubeTracker class for scrapping artists data.""" def __init__(self): logging.basicConfig(level=logging.INFO) self.__yt_api__ = None self.__executor__ = YouTubeStatsSFExecutor(sf_config=config.SF_CONFIG) def get_executor(self): """Return SnowFlake executor.""" return self.__executor__ def get_yt_api(self): """Return YouTube API instance.""" return self.__yt_api__ def create_table(self): """Create tables if not exist and read last week stats.""" exists = ' table already exists.' executor = self.get_executor() query_result = executor.create_video_statistics_table() if 'already exists' in query_result[0]: logging.info('%s %s', config.snowflake_table_names['video_statistics'].upper(), exists) def get_all_the_content_with_next_page_token(self, id_, page_token, items: list): """Recursive method getting all the content using next page token.""" items_ = [] token = None try: comments = self.get_yt_api().get_comment_threads(video_id=id_, page_token=page_token) items_, token = comments.items, comments.nextPageToken except ReadTimeout as _: logging.error("The comments cannot be read.") items.extend(items_) if token: self.get_all_the_content_with_next_page_token(id_, token, items) return items def get_video_info(self, video_link: str): """Return video info: title, publishing date, views, likes etc.""" video_id = get_id_from_link(video_link) video_by_id = self.get_yt_api().get_video_by_id(video_id=video_id).items[0] title = video_by_id.snippet.title description = video_by_id.snippet.description date_published = video_by_id.snippet.publishedAt views = video_by_id.statistics.viewCount or '0' likes = video_by_id.statistics.likeCount or '0' dislikes = video_by_id.statistics.dislikeCount or '0' comments_count = video_by_id.statistics.commentCount or '0' try: comments_ = self.get_all_the_content_with_next_page_token( video_id, None, []) comments = list( map(lambda c: {'comment': c.snippet.topLevelComment.snippet.textDisplay, 'likes': c.snippet.topLevelComment.snippet.likeCount, 'author': c.snippet.topLevelComment.snippet.authorDisplayName, 'authorChannelId': comments_[0].snippet.topLevelComment.snippet.authorChannelUrl.replace( config.YOUTUBE_LINK, '')}, comments_)) except PyYouTubeException as error: if 'has disabled comments' in error.message: logging.info('Comments seem to be disabled.') else: logging.error(error.message) comments = [] video_info = {'video_id': title, 'video_title': title, 'date_published': date_published, 'description': description, 'author': video_by_id.snippet.channelTitle, 'channel_id': video_by_id.snippet.channelId, 'views': views, 'likes': likes, 'dislikes': dislikes, 'comments_count': comments_count, 'comments': '\n'.join(x.get('comment') for x in comments) } return video_info def update_stats(self, video_link, stats: dict): """Update all stats for artist at SnowFlake.""" self.get_executor().insert_video_statistics_table(video_link, stats) logging.info("%s video stats has been inserted.", video_link) def get_and_update_stats(self, video_link): """Update stats for every single artist.""" try: stats = self.get_video_info(video_link) self.update_stats(video_link, stats) except PyYouTubeException as error: if 'you have exceeded your' in error.message: logging.warning('The request cannot be completed because ' 'YouTube quota has been exceeded.') logging.info('The flow will continue scrapping artists data tomorrow' "starting with %s's channel.", video_link) return {'stop': True, 'video': video_link} return {'stop': False} def perform_tracking(self, video_link): """Perform tracking with pyyoutube.""" self.create_table() res = dict(stop=True) counter = 0 while res.get('stop'): try: self.__yt_api__ = Api(api_key=helpers.get_youtube_api_credentials() .get('client_id')[counter]) res = self.get_and_update_stats(video_link) counter += 1 except IndexError: break if __name__ == '__main__': tracker = YouTubeTracker() tracker.perform_tracking(sys.argv[1])