#! /usr/bin/env python import os import sys sys.path.insert(0, '.') import traceback from datetime import datetime import requests import time import random scouts = [ 'capital_bra', 'lisaandlena', 'rebeccamir', 'carodaur', 'kevinbannier_', 'leoobalys', 'sara_llaxx', 'annamariadamm', 'lesotwins', 'juliabeautx', 'imarealmax', 'timschaecker', 'lisamarie_schiffner', 'laurasophie', 'denise.mski', ] from tracker import tiktok, db db.setup_session() print(db.Session.get_bind()) sentinel_date = datetime(2020, 1, 1) # to ensure we start scraping right away user = 'ogutschik' failed_file_name = f'failed_tiktok_scouts.{user}.txt' if os.path.isfile(failed_file_name): with open(failed_file_name, 'r') as fin: already_failed = [x.strip() for x in fin.read().splitlines() if x.strip()] print(f"Found {len(already_failed)} failed users.") else: already_failed = [] def insert_user(scout, update=True): existing = db.TikTokUser.lookup_by_username(scout) if existing: print(f"{scout} is already in db!") if not update: return existing try: return tiktok.upsert_user_by_id(scout, allow_private=False) except Exception as e: print(f"Failed to get data for {scout}") with open(failed_file_name, 'a') as fout: fout.write(scout) fout.write("\n") traceback.print_exc() time.sleep(20) return None if __name__ == "__main__": print(f"-- Bulk tiktok scout adder --") print(f"Adding new scouts to user {user}\n-------\n") print(f"Appending failed users to file {failed_file_name}") consecutive_failures = 0 for scout in scouts: print(f"Adding {scout}") if scout in already_failed: print(f"Skipping known bad username: {scout}") continue added = insert_user(scout) print(f"Inserted, adding to user '{user}'") scout_exists = db.Session.query(db.UsersTtInfluencers).filter(db.UsersTtInfluencers.username == user, db.UsersTtInfluencers.unique_id == scout).first() if added and not scout_exists: db.Session.add(db.UsersTtInfluencers( username=user, unique_id=scout )) db.Session.commit() sleep_secs = 5 print(f"Completed {scout}, sleeping for {sleep_secs}") time.sleep(sleep_secs)