"""Artist Url Model.""" from datetime import datetime from datetime import timedelta from itertools import groupby from oto import response from oto import status import sqlalchemy from sqlalchemy import and_ from sqlalchemy import or_ from social_analytics.connectors import loggly from social_analytics.connectors import mysql from social_analytics.connectors import sentry from social_analytics.constants import collectors from social_analytics.constants import error from social_analytics.models.social_profile import SocialProfile logger = loggly.get_current_logger() class ArtistUrl(mysql.BaseModel): """Artist URL model.""" __tablename__ = 'artist_url' url_id = sqlalchemy.Column( sqlalchemy.INT, primary_key=True, autoincrement=True) artist_id = sqlalchemy.Column(sqlalchemy.INT) social_profile_id = sqlalchemy.Column(sqlalchemy.INT) site_id = sqlalchemy.Column(sqlalchemy.INT) url = sqlalchemy.Column(sqlalchemy.VARCHAR(45)) evaluated_for_collection = sqlalchemy.Column( sqlalchemy.Boolean, default=False) def to_dict(self): """Return the object as dictionary.""" return dict( url_id=self.url_id, artist_id=self.artist_id, social_profile_id=self.social_profile_id, site_id=self.site_id, url=self.url, evaluated_for_collection=self.evaluated_for_collection) @mysql.autosession() def get_artist_urls_with_artist_id(artist_id, session): """Get all the artist urls with artist_id. Args: artist_id (int): The artist_id that we want to retrieve artist_urls. session (Session): The mysql session. Returns: response.Response: containing the created social profile dict. """ if not artist_id: return response.create_error_response( status.BAD_REQUEST, error.ERROR_MESSAGE_EMPTY_BODY) artist_urls = session.query(ArtistUrl).filter_by(artist_id=artist_id).all() results = [row.to_dict() for row in artist_urls] if len(results) > 0: return response.Response({'items': results}) return response.create_not_found_response() @mysql.autosession() def get_artist_url_with_artist_id_and_site_id(artist_id, site_id, session): """Get all the artist urls with artist_id. Args: artist_id (int): The artist_id that we want to retrieve artist_url. site_id (int): The site_id e.g. 2 for Facebook. session (Session): The mysql session. Returns: response.Response: containing the social profile dict. """ if not artist_id or not site_id: return response.create_error_response( status.BAD_REQUEST, error.ERROR_MESSAGE_EMPTY_BODY) artist_url = session.query(ArtistUrl).filter_by( artist_id=artist_id, site_id=site_id).first() if artist_url: return response.Response(artist_url.to_dict()) return response.create_not_found_response() @mysql.autosession() def create_artist_url(data, session): """ Create a new Artist Url. Args: data (dict): the data from which to create the artist_url. session (Session): the mysql session. Returns: response.Response: containing the created artist_url dict. """ if not data: return response.create_error_response( status.BAD_REQUEST, error.ERROR_MESSAGE_EMPTY_BODY) artist_url = ArtistUrl(**data) session.add(artist_url) session.commit() return response.Response(artist_url.to_dict()) @mysql.autosession() def update_artist_url(data, session): """Update artist url. Args: data (dict): the data from which to update the artist_url. session (Session): the mysql session. Returns: response.Response: containing the updated artist_url dict. """ if not data: return response.create_error_response( status.BAD_REQUEST, error.ERROR_MESSAGE_EMPTY_BODY) sentry_client = sentry.get_client() logger.info( 'ARTIST URL: About to create artist url with data {0}'.format(data)) artist_url = ArtistUrl(**data) logger.info( 'ARTIST URL: Artist url created {0}'.format(artist_url)) session.query( ArtistUrl).filter( ArtistUrl.url_id == artist_url.url_id).update( data) session.commit() logger.info( 'ARTIST URL: Artist url saved {0}'.format(artist_url.to_dict())) sentry_client.captureMessage( message='Status of artist url migration', stack=True, extra={ 'message': 'Artist url sync: artist_url updated {0}'.format( artist_url.to_dict()), 'status': 200}) return response.Response(artist_url.to_dict()) @mysql.autosession() def get_social_profile_ids_by_artist_ids(artist_ids, session): """Get social profile ids by artist ids. Args: artist_ids (list): The artist ids. session (Session): The MySQL session. Returns: response.Response: Containing a list of social profile ids. """ if not artist_ids: return response.create_error_response( status.BAD_REQUEST, error.ERROR_MESSAGE_BAD_PARAMS) rows = ( session.query(ArtistUrl, SocialProfile).join( SocialProfile, SocialProfile.social_profile_id == ArtistUrl .social_profile_id).filter( ArtistUrl.artist_id.in_(artist_ids)).all()) items = [] for artist_id, artist_profile_group in groupby( rows, lambda row: row[0].artist_id): profiles = [group[1] for group in artist_profile_group] profile_dicts = [profile.to_dict() for profile in profiles] items.append( { 'artist_id': artist_id, 'social_profiles': profile_dicts } ) if not items: return response.create_not_found_response() return response.Response({'items': items}) @mysql.autosession() def clear_social_profile(social_profile_id, artist_id, session): """Clear an Artist Url social_profile_id column. Args: social_profile_id (int): The social profile id. artist_id (int): The artist id. Returns: response.Response: containing the deleted artist_url dicts. """ if not social_profile_id or not artist_id: return response.create_error_response( status.BAD_REQUEST, error.ERROR_MESSAGE_BAD_PARAMS) rows = session.query( ArtistUrl).filter( ArtistUrl.social_profile_id == social_profile_id, ArtistUrl.artist_id == artist_id).all() if not rows: return response.create_not_found_response() for row in rows: row.social_profile_id = None session.commit() items = [row.to_dict() for row in rows] return response.Response({'items': items}) @mysql.autosession() def get_linked_artist_urls(artist_ids, session): """Return all the artist urls for given artist ids. Args: artist_ids (list): A list of artist ids. session (Session): The mysql db session. Returns: response.Response: containing a list of artist ids. """ if not artist_ids: return response.create_error_response( status.BAD_REQUEST, error.ERROR_MESSAGE_BAD_PARAMS) rows = session.query( ArtistUrl.artist_id).filter(ArtistUrl.artist_id.in_(artist_ids)).all() if rows: ids = [result.artist_id for result in rows] return response.Response({'items': ids}) return response.create_not_found_response() @mysql.autosession() def get_artist_ids_that_have_profiles(artist_ids, session): """Return all the artist ids that are associated with social profiles. Args: artist_ids (list): A list of artist ids. session (Session): The mysql db session. Returns: response.Response: containing a list of artist ids. """ if not artist_ids: return response.create_error_response( status.BAD_REQUEST, error.ERROR_MESSAGE_BAD_PARAMS) rows = session.query( ArtistUrl.artist_id).filter(ArtistUrl.artist_id.in_(artist_ids)).all() if rows: ids = [result.artist_id for result in rows] return response.Response({'items': ids}) return response.create_not_found_response() @mysql.autosession() def get_artist_urls_for_syncing(session): """Return all the artist urls that need to be synced. This applies to any artist_url rows that have not been created through ows-social-analytics and must be updated to be valid for use by ows-social-analytics. Returns: response.Response: Containing a list of artist_urls """ # It should match either this or one of the current two like patterns. rows = ( session.query(ArtistUrl) .filter(ArtistUrl.site_id == 2) .filter(ArtistUrl.evaluated_for_collection == 0) .filter(or_( or_(ArtistUrl.url.like('%profile.php?id=%'), ArtistUrl.url.like('%pages/%/%'))), ArtistUrl.url.op('regexp')('.*-[0-9]+')) .filter() .limit(100)) if rows: items = [row.to_dict() for row in rows] return response.Response({'items': items}) return response.create_not_found_response() @mysql.autosession() def get_social_profiles_for_collection(session): """Get all social profiles that are eligible for collection. Query the db for any social profiles in collection slots within the time window. The time window starts from the current time slot and goes back an amount of time slots specified by collectors.TIME_SLOTS_WINDOW. There are two cases: 1) The current time slot is greater or equal to our time window size, so e.g. -with a time window size of 12 -we are currently at time slot 13 - so query everything from (but not including) 1 to 13 (included). 2) The current time slot is less than our time window size, so e.g. -with a time window size of 12 -we are currently at time slot 9 -so query everything from 0 (included) to 9 (included) OR everything from 285 (not included) to 288 (included). Args: session (Session): the mysql session. Returns: response.Response: containing a list of dictionaries that can be collected now. e.g. { 'social_profile_id': 1, 'platform': 'facebook', 'platform_id': '123a' } """ time_slots = collectors.TIME_SLOTS time_slots_window = collectors.TIME_SLOTS_WINDOW current_time_slot = calculate_current_time_slot() logger.info('TIMESLOTS: The current time slot is: {}'.format( current_time_slot)) twenty_hours_ago = datetime.now() - timedelta(hours=20) rows = ( session .query( SocialProfile.platform, SocialProfile.platform_id, ArtistUrl.social_profile_id) .distinct() .join( ArtistUrl, SocialProfile.social_profile_id == ArtistUrl.social_profile_id)) if current_time_slot >= time_slots_window: logger.info('TIMESLOTS: About to subtract 12 time slots') starting_time_slot = current_time_slot - time_slots_window logger.info('TIMESLOTS: The starting time slot is: {0}'.format( starting_time_slot)) rows = ( rows.filter( ArtistUrl.artist_id.__mod__(time_slots) > starting_time_slot) .filter( ArtistUrl.artist_id.__mod__(time_slots) <= current_time_slot)) else: starting_time_slot = ( time_slots - (time_slots_window - current_time_slot)) logger.info('TIMESLOTS: about to subtract {0} time slots'.format( starting_time_slot)) logger.info('TIMESLOTS: The starting time slot is: {0}'.format( starting_time_slot)) rows = ( rows.filter( or_( and_( ArtistUrl.artist_id.__mod__(time_slots) >= 0, ArtistUrl.artist_id.__mod__(time_slots) <= current_time_slot), and_( ArtistUrl.artist_id.__mod__(time_slots) > starting_time_slot, ArtistUrl.artist_id.__mod__(time_slots) <= time_slots)) )) rows = ( rows.filter( or_( SocialProfile.last_collected_date <= twenty_hours_ago, SocialProfile.last_collected_date.is_(None)))) logger.info('TIMESLOTS SQL: {0}'.format(str(rows.statement))) rows = rows.all() results = [{ 'social_profile_id': row.social_profile_id, 'platform': row.platform, 'platform_id': row.platform_id, } for row in rows] if len(results) > 0: return response.Response({'items': results}) return response.create_not_found_response() def calculate_current_time_slot(): """Calculate the current time slot. Returns: An integer from 0 to a max of constants.collectors.TIME_SLOT - 1 representing the time slot in day. """ # Use floor division to get integer time_slot_step = (24 * 60 * 60) // collectors.TIME_SLOTS now = datetime.now() midnight = now.replace(hour=0, minute=0, second=0, microsecond=0) seconds = (now - midnight).seconds return seconds // time_slot_step