"""SoundCloud scrapping flow.""" import logging import sys import time from bs4 import BeautifulSoup from lxml import etree from selenium import webdriver from snowflake_executor import SoundCloudStatsSFExecutor import config def get_id_from_link(text): """Return SoundCloud video link from config.""" return text. \ replace(config.SOUNDCLOUD_LINK, '') def scroll_down(driver_, last_height=0): """Scroll a page down.""" driver_.execute_script("window.scrollTo(0, document.body.scrollHeight);") time.sleep(2) height = driver_.execute_script("return document.body.scrollHeight") if last_height == height: return scroll_down(driver_, height) def set_webdriver(): """Set class' webdriver argument.""" options = webdriver.ChromeOptions() options.add_argument('headless') options.add_argument('--disable-dev-shm-usage') options.add_argument('--no-sandbox') options.add_argument('--single-process') options.add_argument('--disable-extensions') options.add_argument('disable-infobars') options.add_argument('start-maximized') driver = webdriver.Chrome('/soundcloud-track-statistics/chromedriver', options=options) return driver class SoundCloudTracker: """SoundCloudTracker class for scrapping artists data.""" def __init__(self): logging.basicConfig(level=logging.INFO) self.__executor__ = SoundCloudStatsSFExecutor(sf_config=config.SF_CONFIG) self.__driver__ = set_webdriver() def get_executor(self): """Return SnowFlake executor.""" return self.__executor__ def get_driver(self): """Return Selenium webdriver.""" return self.__driver__ def create_table(self): """Create tables if not exist and read last week stats.""" exists = ' table already exists.' executor = self.get_executor() query_result = executor.create_track_statistics_table() if 'already exists' in query_result[0]: logging.info('%s %s', config.snowflake_table_names['track_statistics'].upper(), exists) def get_dom_of_artist_page(self, track_link): """Return DOM of artist's SC page.""" driver = self.get_driver() driver.get(track_link) scroll_down(driver) soup = BeautifulSoup(driver.page_source, 'html.parser') html = soup.prettify() dom = etree.HTML(html) return dom def get_track_info(self, track_link): """Return track info: title, publishing date, views, likes etc.""" def get_dom(): dom_ = self.get_dom_of_artist_page(track_link) artist_info_ = dom_.xpath(config.xpaths.get('artist_info'))[0] return dom_, artist_info_ try: dom, artist_info = get_dom() except IndexError: logging.error('Firstly you have to accept all cookies.') self.get_driver().implicitly_wait(10) self.get_driver().find_element_by_xpath( config.xpaths.get('cookies')).click() dom, artist_info = get_dom() artist = artist_info.text.replace('\n', '').lstrip().rstrip() artist_link = config.SOUNDCLOUD_LINK + artist_info.attrib.get('href') title = dom.xpath(config.xpaths.get('title'))[0].text.replace('\n', '').lstrip().rstrip() plays = int(dom.xpath(config.xpaths.get('plays') )[0].text.replace('\n', '').replace('plays', '').replace(',', ''). lstrip().rstrip()) likes = int(dom.xpath(config.xpaths.get('likes'))[0].text.replace('\n', ''). replace('plays', '').replace(',', '').lstrip().rstrip()) reposts = int(dom.xpath(config.xpaths.get('reposts'))[0].text.replace('\n', ''). replace('reposts', '').replace(',', '').lstrip().rstrip()) comments = [c.text.replace('\n', '').lstrip().rstrip() for c in dom.xpath(config.xpaths.get('comments'))] track_info = { 'track_title': title, 'track_link': track_link, 'artist': artist, 'artist_link': artist_link, 'plays': plays, 'likes': likes, 'reposts': reposts, 'comments_count': len(comments), 'comments': '\n'.join(comments) } return track_info def update_stats(self, video_link, stats: dict): """Update all stats for artist at SnowFlake.""" self.get_executor().insert_video_statistics_table(stats) logging.info("%s video stats has been inserted.", video_link) def get_and_update_stats(self, video_link): """Update stats for every single artist.""" stats = self.get_track_info(video_link) self.update_stats(video_link, stats) def perform_tracking(self, video_link): """Perform tracking with selenium.""" self.create_table() self.get_and_update_stats(video_link) if __name__ == '__main__': tracker = SoundCloudTracker() tracker.perform_tracking(sys.argv[1])