import re from datetime import datetime from flask_restful import Api import flask from werkzeug.exceptions import Unauthorized from tracker import db, spotify from api.resources import AuthenticatedResource from api import api_errors from tracker.utils import null_safe_path from tracker.youtube import youtube_url_translator from tracker import tiktok from tracker.logger import get_logger from .users import AdministrateUsers from . import active_artists logger = get_logger(__name__) def register_urls(api: Api, root="/api/admin"): api.add_resource(SpotifyPlaylists, root + "/spotify_playlists") api.add_resource(SpotifyPlaylistEdit, root + "/spotify_playlist/edit") api.add_resource(SpotifyPlaylistCheck, root + "/spotify_playlist/check") api.add_resource(SpotifyPlaylistAdd, root + "/spotify_playlist/add") api.add_resource(YouTubeChannels, root + "/youtube_channels") api.add_resource(YouTubeChannelEdit, root + "/youtube_channel/edit") api.add_resource(YouTubeChannelCheck, root + "/youtube_channel/check") api.add_resource(YouTubeChannelAdd, root + "/youtube_channel/add") api.add_resource(SoundCloudInfluencers, root + "/soundcloud_influencers") api.add_resource(SoundCloudInfluencerEdit, root + "/soundcloud_influencer/edit") api.add_resource(SoundCloudInfluencerCheck, root + "/soundcloud_influencer/check") api.add_resource(SoundCloudInfluencerAdd, root + "/soundcloud_influencer/add") api.add_resource(TwitterInfluencers, root + "/twitter_influencers") api.add_resource(TwitterInfluencerEdit, root + "/twitter_influencer/edit") api.add_resource(TwitterInfluencerCheck, root + "/twitter_influencer/check") api.add_resource(TwitterInfluencerAdd, root + "/twitter_influencer/add") api.add_resource(InstagramInfluencers, root + "/instagram_influencers") api.add_resource(InstagramInfluencerEdit, root + "/instagram_influencer/edit") api.add_resource(InstagramInfluencerCheck, root + "/instagram_influencer/check") api.add_resource(InstagramInfluencerAdd, root + "/instagram_influencer/add") api.add_resource(TikTokInfluencers, root + "/tiktok_influencers") api.add_resource(TikTokInfluencerEdit, root + "/tiktok_influencer/edit") api.add_resource(TikTokInfluencerCheck, root + "/tiktok_influencer/check") api.add_resource(TikTokInfluencerAdd, root + "/tiktok_influencer/add") api.add_resource(AdministrateUsers, root + "/users") active_artists.register_urls(api, root) class ListEndpoint(AuthenticatedResource): # <[pk], scout_display_name, active, category> sql_query = None def get(self): return list( db.execute_to_dicts(self.sql_query, params=dict(username=self.user.name)) ) class EditEndpoint(AuthenticatedResource): pk_field: str = None db_class = None db_permission_class = None def post(self): data, record = self._get_and_verify_data() record.category = data["category"] db.Session.commit() return data def delete(self): data, record = self._get_and_verify_data() db.Session.delete(record) db.Session.commit() return data def _get_and_verify_data(self): data = flask.request.json if not data.get(self.pk_field): raise api_errors.BadRequestError record = ( db.Session.query(self.db_permission_class) .filter_by( **{self.pk_field: data[self.pk_field], "username": self.user.name} ) .first() ) if not record: raise api_errors.NotFoundError return data, record class AddEndpoint(AuthenticatedResource): def _get_for_username(self): json_data = flask.request.json if "username" in json_data and json_data["username"]: if not self.user.is_admin: raise Unauthorized for_username = json_data["username"] is_in_db = db.Session.query(db.Users).get(for_username) if not is_in_db: api_errors.BadRequestError( "No user found for username: {}".format(for_username) ) else: for_username = self.user.name return for_username class SpotifyPlaylists(ListEndpoint): sql_query = """ select usp.spyid, spy_playlist.name as scout_display_name, usp.category, spy_playlist.active from spy_playlist join users_spy_playlists usp on usp.spyid = spy_playlist.spyid and username = :username """ class SpotifyPlaylistEdit(EditEndpoint): pk_field = "spyid" db_class = db.SpyPlaylist db_permission_class = db.UsersSpyPlaylists class SpotifyPlaylistCheck(AuthenticatedResource): def post(self): data = flask.request.json url = data["url"] playlist = spotify.decode_url_or_uri(url) if playlist: check = spotify.get_playlist(None, playlist, basic_fields=True) if check: return { "preview_text": check["name"], "thumbnail": check["images"][0]["url"] if ("images" in check and len(check["images"]) > 0) else None, } raise api_errors.NotFoundError def upsert_spotify_playlist(username, url_or_uri): playlist = spotify.decode_url_or_uri(url_or_uri) if not playlist: raise api_errors.NotFoundError playlist_data = spotify.get_playlist( user=None, playlist_id=playlist, basic_fields=True ) if not playlist_data or playlist_data.get("name") is None: raise api_errors.NotFoundError spotify._upsert_playlist(None, playlist_data["id"], playlist_data["name"]) existing_record = ( db.Session.query(db.UsersSpyPlaylists.spyid) .filter_by(spyid=playlist, username=username) .first() ) if not existing_record: db.Session.add(db.UsersSpyPlaylists(spyid=playlist, username=username)) db.Session.commit() class SpotifyPlaylistAdd(AddEndpoint): def post(self): data = flask.request.json url = data["url"] for_username = self._get_for_username() upsert_spotify_playlist(for_username, url) return "", 201 class YouTubeChannels(ListEndpoint): sql_query = """ select uyc.ytid, yt_channels.name as scout_display_name, active, uyc.category from yt_channels join users_yt_channels uyc on uyc.ytid = yt_channels.ytid and username = :username """ class YouTubeChannelEdit(EditEndpoint): pk_field = "ytid" db_class = db.YtChannel db_permission_class = db.UsersYtChannels def _get_channel_from_youtube_api(url) -> youtube_url_translator.ChannelInfo: if "/" not in url: # doesn't appear to be a youtube url if re.search(r"^[a-zA-Z0-9\_\-]$", url): # appears to be a channel id... url = f"https://www.youtube.com/channel/{url}" try: channels = youtube_url_translator.get_channels_from_url(url) except RuntimeError as e: logger.exception(f"Failed to get youtube channel info for url {url}, {e}") raise api_errors.NotFoundError( "That doesn't seem to be a youtube page, please try the link again" ) if not channels: raise api_errors.NotFoundError( "That doesn't seem to be a youtube page, please try the link again" ) return channels[0] class YouTubeChannelCheck(AuthenticatedResource): def post(self): channel_data = _get_channel_from_youtube_api(flask.request.json["url"]) return dict( id=channel_data.ytid, name=channel_data.title, username="", avatar=channel_data.thumbnail, source_url="https://www.youtube.com/channel/" + channel_data.ytid, ) class YouTubeChannelAdd(AddEndpoint): def post(self): channel_data = _get_channel_from_youtube_api(flask.request.json["url"]) for_username = self._get_for_username() ytid = channel_data.ytid name = channel_data.title exists = db.Session.query(db.YtChannel).get(ytid) if not exists: db.Session.add(db.YtChannel(ytid=ytid, name=name)) db.Session.flush() exsting_record = ( db.Session.query(db.UsersYtChannels) .filter_by(ytid=ytid, username=for_username) .first() ) if not exsting_record: db.Session.add(db.UsersYtChannels(ytid=ytid, username=for_username)) db.Session.commit() return "", 201 class SoundCloudInfluencers(ListEndpoint): sql_query = """select usi.scid, sc_users.name as scout_display_name, usi.category, sc_users.active from sc_users join users_sc_influencers usi on usi.scid = sc_users.scid and username = :username""" class SoundCloudInfluencerEdit(EditEndpoint): pk_field = "scid" db_class = db.ScUser db_permission_class = db.UsersScInfluencers def _fetch_user_from_soundcloud(data): url = data["url"] if not url or "soundcloud" not in url: raise api_errors.BadRequestError( "Expect a soundcloud page url like https://soundcloud.com/" ) from tracker import soundcloud user = soundcloud.resolve_user(url) if user: return user raise api_errors.NotFoundError class SoundCloudInfluencerCheck(AuthenticatedResource): def post(self): user = _fetch_user_from_soundcloud(flask.request.json) return { "preview_text": user.get("full_name") or user.get("username"), "thumbnail": user.get("avatar_url"), } class SoundCloudInfluencerAdd(AddEndpoint): def post(self): user = _fetch_user_from_soundcloud(flask.request.json) for_username = self._get_for_username() if user: from tracker import soundcloud soundcloud.add_influencer_from_url( flask.request.json["url"], usernames=[for_username] ) return "", 201 raise api_errors.NotFoundError # Twitter class TwitterInfluencers(ListEndpoint): sql_query = ( "select tracked_entities.eid, tracked_entities.name, tracked_entities.active, uti.category " "from tracked_entities " "join users_tw_influencers uti on uti.eid = tracked_entities.eid and username = :username " "where twitter_id is not null order by uti.added desc" ) class TwitterInfluencerEdit(EditEndpoint): pk_field = "eid" db_class = db.TrackedEntity db_permission_class = db.UsersTwInfluencers class TwitterInfluencerCheck(AuthenticatedResource): def post(self): url = flask.request.json["url"] if url.startswith("@"): username = url[1:] else: if not url or "twitter" not in url: raise api_errors.BadRequestError( "Expect a twitter page url like https://twitter.com/" ) username = url.split("/")[-1] from tracker import twitter user = twitter.new_instance().fetch_user(username, shallow=True) if user: return { "preview_text": user.get("screen_name"), "thumbnail": user.get("profile_image_url"), } raise api_errors.NotFoundError class TwitterInfluencerAdd(AddEndpoint): def post(self): url = flask.request.json["url"] for_username = self._get_for_username() if url.startswith("@"): username = url[1:] else: if not url or "twitter" not in url: raise api_errors.BadRequestError( "Expect a twitter page url like https://twitter.com/" ) username = url.split("/")[-1] from tracker import twitter, actions user = twitter.new_instance().fetch_user(username, shallow=True) if user: print("*** Adding Twitter user", db.Session.get_bind()) dbrecord = ( db.Session.query(db.TrackedEntity) .filter_by(twitter_id=user["id_str"]) .first() ) if dbrecord: dbrecord.twitter_data = user dbrecord.is_influencer = True else: dbrecord = actions.add_entity(twitter_data=user, is_influencer=True) # Want to ensure this gets picked up by the nightly tracker dbrecord.twitter_last_update = None db.Session.flush() db.Session.add( db.UsersTwInfluencers(username=for_username, eid=dbrecord.eid) ) db.Session.commit() return "", 201 raise api_errors.NotFoundError # Instagram class InstagramInfluencers(ListEndpoint): sql_query = """select in_user.inid, coalesce(full_name, uii.inid) as scout_display_name, uii.category, in_user.active from in_user join users_in_influencers uii on uii.inid = in_user.inid and username = :username""" class InstagramInfluencerEdit(EditEndpoint): pk_field = "inid" db_class = db.InUser db_permission_class = db.UsersInInfluencers def _parse_inid_from_request(value): valid_insta_name_re = re.compile(r"^[a-zA-Z0-9\_\.]{2,30}$") # in case there's something like '?modal=true' included: value = value.split("?")[0].split("#")[0] # remove a potential trailing slash value = value.strip().strip("/") if "/" in value: # looks like a url... if "instagram." not in value: raise api_errors.BadRequestError( "Expect a instagram page url like https://www.instagram.com/ or an @" ) value = value.split("/")[-1] if value.startswith("@"): value = value[1:] value = db.InUser.canonicalize_inid(value) if not valid_insta_name_re.search(value): raise api_errors.BadRequestError( f"{value} does not look like a valid insta user, it should be 2-30 letters or numbers (or a . or _)" ) return value.lower() def _make_insta_profile_result(value): inid = _parse_inid_from_request(value) existing: db.InUser = db.Session.query(db.InUser).get(inid) if existing and existing.page_data: user = null_safe_path( existing.page_data, "entry_data", "ProfilePage", 0, "user" ) return ( None if user is None else dict( id=inid, name=user.get("full_name") or user.get("username"), username=user.get("username"), source_url="https://www.instagram.com/" + user.get("username") + "/", avatar=user.get("profile_pic_url"), ) ) # rate limiting means we can't really validate this anymore. we include the 'validate_url' which # the user is encouraged to check themselves. return dict( id=inid, name=inid, username=inid, source_url="https://www.instagram.com/" + inid + "/", validate_url="https://www.instagram.com/" + inid + "/", avatar=None, ) class InstagramInfluencerCheck(AuthenticatedResource): def post(self): url = flask.request.json["url"] return _make_insta_profile_result(url) class InstagramInfluencerAdd(AddEndpoint): def post(self): url = flask.request.json["url"] for_username = self._get_for_username() insta_username = _parse_inid_from_request(url) return add_insta_influencer(for_username, insta_username) def add_insta_influencer(for_username, insta_username): insta_username = insta_username.lower() existing = db.Session.query(db.InUser).get(insta_username) if existing: existing.is_influencer = True else: db.Session.add(db.InUser(inid=insta_username, is_influencer=True)) db.Session.add(db.UsersInInfluencers(inid=insta_username, username=for_username)) db.Session.commit() return "", 201 # TikTok! class TikTokInfluencers(ListEndpoint): # we won't create the TikTokUser record so we do a left join. sql_query = """ select unique_id, coalesce(nickname, unique_id) as "scout_display_name", coalesce(tiktok_users.active, '1') as "active", ui.category from users_tt_influencers ui left join tiktok_users using (unique_id) where ui.username = :username """ class TikTokInfluencerEdit(EditEndpoint): pk_field = "unique_id" db_class = db.TikTokUser db_permission_class = db.UsersTtInfluencers class TikTokInfluencerCheck(AuthenticatedResource): def post(self): url = flask.request.json["url"] user_unique_id = _parse_tiktok_username(url) exists = db.TikTokUser.lookup_by_username(user_unique_id) if exists: return dict( id=exists.ttid, name=exists.nickname, username=exists.unique_id, source_url="https://www.tiktok.com/share/user/" + exists.ttid, avatar=exists.cover_image_url, ) try: info: tiktok.TikTokUserData = tiktok.get_user_info_from_username( user_unique_id ) except tiktok.UnkownTikTokUserError as e: raise api_errors.NotFoundError( info=f"No tiktok user found with the username {user_unique_id}" ) return dict( id=info.ttid, name=info.nickname, username=info.unique_id, source_url=f"https://www.tiktok.com/share/user/{info.ttid}", avatar=info.cover_image_url, ) class TikTokInfluencerAdd(AddEndpoint): def post(self): url = flask.request.json["url"] for_username = self._get_for_username() user_unique_id = _parse_tiktok_username(url) existing = db.TikTokUser.lookup_by_username(user_unique_id) if not existing: try: existing = tiktok.upsert_user_by_id(user_unique_id, allow_private=False) except tiktok.PrivateTikTokAccountError as e: raise api_errors.BadRequestError( f"{user_unique_id} is a private account and cannot be used for scouting." ) else: if not existing: raise api_errors.BadRequestError(f"{user_unique_id} was not found") existing.is_influencer = True existing.last_visited = datetime(2020, 1, 1) # ensure it's scraped on the day db.Session.add( db.UsersTtInfluencers(unique_id=user_unique_id, username=for_username) ) db.Session.commit() return "", 201 def _parse_tiktok_username(url): if "@" in url: # nb: some non-english tiktok usernames are the secUid (long ugly string) # and those are case sensitive user_unique_id = url.split("@")[1].split("?")[0].strip() else: user_unique_id = url.strip() if "/" in user_unique_id: raise api_errors.BadRequestError( "That doesn't look like a valid tiktok username. Please provide the url from https://www.tiktok.com" ) return user_unique_id