#! /usr/bin/env python import sys from contextlib import contextmanager import time sys.path.insert(0, ".") from datetime import datetime import time import requests from pymysql.cursors import Cursor import pymysql from tracker import soundcloud from tracker.youtube import youtube from tracker.config import _get_secret_json import traceback unicron_conf_dict = _get_secret_json("unicronSecretsProd") mysql_kwargs = dict( host=unicron_conf_dict["UC_MYSQL_HOST"], user="unicron", password=unicron_conf_dict["UC_MYSQL_PASSWORD"], db="unicron", charset="utf8mb4", # cursorclass='DictCursor' ) SC_TABLE_NAME = "jimmy_sc_plays" SC_TRACK1 = None SC_TRACK2 = None SC_TRACK3 = None YT_TABLE_NAME = "switch_youtubes" YT_VIDEO_1 = "wFU1exmLi8E" YT_VIDEO_2 = "oAlmKyVEHSM" YT_VIDEO_3 = "wz-nX4jSdiA" YT_VIDEO_4 = "qxftBHyA5qA" YT_VIDEO_5 = "HSKEcicmXFk" # TIKTOK_TABLE_NAME = 'jimmy_tiktoks' # TIKTOK_URL1 = 'https://www.tiktok.com/music/Jimmy-6905062735650916354' # TIKTOK_URL2 = None # TIKTOK_URL3 = None class fakecur: execute = lambda *args: print( "Would execute\n\n", args[0], "\n\n", args[1] if len(args) > 1 else "" ) @contextmanager def commit_after() -> Cursor: connection = pymysql.connect(**mysql_kwargs) try: with connection.cursor() as cur: yield cur connection.commit() finally: try: connection.close() except Exception as e: print("Warning, failed to close connection") print(e) def capture_soundcloud(): if not SC_TRACK1: print("No SoundCloud") return track_id_strs = [str(i) for i in (SC_TRACK1, SC_TRACK2, SC_TRACK3) if i is not None] try: datalist = list( sorted( soundcloud.client.get("/tracks", limit=5, ids=",".join(track_id_strs)), key=lambda r: r.id, ) ) except Exception as e: print(f"Failed to fetch soundcloud data {e}") traceback.print_exc() return columns = ["ts"] args = [datetime.utcnow().isoformat()] for i, data in enumerate(datalist, start=1): columns.extend([f"plays_{i}", f"comments_{i}", f"views_{i}"]) args.extend( [ data.playback_count, data.comment_count, data.favoritings_count, ] ) pcts = ["%s"] * len(columns) sql = f"""insert into {SC_TABLE_NAME} ({",".join(columns)}) values ({",".join(pcts)})""" with commit_after() as cur: print("Recording soundcloud", args) cur.execute(sql, args) def capture_youtubes(): all_ytids = list( filter( None, [ YT_VIDEO_1, YT_VIDEO_2, YT_VIDEO_3, YT_VIDEO_4, YT_VIDEO_5, ], ) ) if not all_ytids: print("No youtubes") return try: items = list( sorted( youtube._fetch_video_stats(video_ids=all_ytids)["items"], key=lambda x: all_ytids.index(x["id"]), ) ) except Exception as e: print(f"Failed to fetch youtube data {e}") traceback.print_exc() return columns = ["ts"] args = [datetime.utcnow().isoformat()] for i, item in enumerate(items, start=1): columns.extend([f"views{i}", f"likes{i}", f"dislikes{i}", f"comments{i}"]) args.extend( [ item["statistics"]["viewCount"], item["statistics"]["likeCount"], item["statistics"]["dislikeCount"], item["statistics"]["commentCount"], ] ) pcts = ["%s"] * len(columns) sql = f"""insert into {YT_TABLE_NAME} ({",".join(columns)}) values ({",".join(pcts)})""" with commit_after() as cur: print("Recording youtube", args) cur.execute(sql, args) def main_loop(scrape_period_seconds): starttime = time.time() while True: # capture_soundcloud() capture_youtubes() # capture_tiktoks() time.sleep( scrape_period_seconds - ((time.time() - starttime) % scrape_period_seconds) ) import json if __name__ == "__main__": scrape_period_seconds = 20.0 * 60.0 print(f"Launching switch youtubes, every {scrape_period_seconds} seconds") main_loop(scrape_period_seconds)