"""Copy track playlist history data for today or up to a week ago.""" from datetime import date, datetime, timedelta import sentry_sdk from sentry_sdk.utils import BadDsn import config from logger import logger import main_db from redis_db import get_redis_lock import utils try: sentry_sdk.init(dsn=config.SENTRY_DSN, environment=config.ENVIRONMENT) except BadDsn: pass @utils.timing def drop_excess_data(): """Drop track playlist history records that older than a week. """ min_history_date = datetime.utcnow().date() - timedelta(config.MAX_HISTORY_INTERVAL) logger.debug(f"Removing records older {min_history_date}.") row_count = main_db.delete_old_history(min_history_date) logger.debug(f"{row_count} records older {min_history_date} were removed.") @utils.timing def process_one_date(history_date: date): """Copy data for specific date. Args: history_date: Date to copy data for. """ logger.debug(f"Copying records for {history_date.isoformat()}") previous_playlist_id = "" count = 1 while True: playlist_ids = main_db.get_playlist_id_chunk(previous_playlist_id, config.PLAYLIST_CHUNK_SIZE) if not playlist_ids or not playlist_ids[0]: break current_playlist_id = utils.max_playlist_id(playlist_ids) rows_count = main_db.copy_history_chunk(history_date, playlist_ids) logger.debug( f"{datetime.utcnow().isoformat()}: {count} {previous_playlist_id}-{current_playlist_id} {rows_count} rows." ) if len(playlist_ids) < config.PLAYLIST_CHUNK_SIZE: break previous_playlist_id = current_playlist_id count = count + 1 def process_history_data(): """Copy history data for today or up to a week ago. """ lock = None try: if config.REDIS_LOCK_ENABLED: # set lock to avoid possible race condition lock = get_redis_lock() if not lock.acquire(blocking=False): logger.info("Can not get lock") return drop_excess_data() current_date = main_db.get_last_date() if not current_date: current_date = datetime.utcnow().date() - timedelta(days=config.MAX_HISTORY_INTERVAL) to_date = datetime.utcnow().date() while current_date <= to_date: process_one_date(current_date) current_date = current_date + timedelta(days=1) except Exception as ex: sentry_sdk.capture_exception(ex) raise finally: if config.REDIS_LOCK_ENABLED and lock: lock.release() if __name__ == "__main__": process_history_data()