"""Syncs newly added artist urls.""" from datetime import datetime from multiprocessing.pool import ThreadPool import re from social_analytics.connectors import sentry from social_analytics.constants import error from social_analytics.logic import facebook_collector from social_analytics.logic import instagram_collector from social_analytics.logic import spotify_collector from social_analytics.models import artist_url as artist_url_model from social_analytics.models import social_profile def sync_artist_urls(): """Sync newly added artist_urls.""" url_to_sync_sync_response = artist_url_model.get_artist_urls_for_syncing() if not url_to_sync_sync_response: return artist_urls = url_to_sync_sync_response.message['items'] tokens = get_tokens() # Create an object for each artist url that contains tokens sync_params = [ { 'artist_url': url, 'tokens': tokens } for url in artist_urls ] with ThreadPool(40) as pool: processed_urls = [] errors = [] try: # Sync each url with ows-social-analytics processed_urls = pool.map(sync_url, sync_params) errors = list( filter(lambda x: 'error' in x.keys(), processed_urls)) except Exception as exc: errors.append(str(exc)) sentry_client = sentry.get_client() sentry_client.captureMessage( message='Errors during artist_url sync', stack=True, extra={ 'message': 'Artist url sync errors', 'errors': errors, 'status': 204}) results = { 'expected_urls_count': len(processed_urls), 'actual_urls_count': (len(processed_urls) - len(errors)), 'urls_data': processed_urls, 'errors_count': len(errors), 'errors_logging': errors } return results def get_tokens(): """"Get social platform API tokens.""" facebook = facebook_collector.refresh_token() instagram = instagram_collector.refresh_token() spotify = spotify_collector.refresh_token()['access_token'] return { 'facebook': facebook, 'instagram': instagram, 'spotify': spotify } def sync_url(params): """Sync a new artist_url. Args: params (dict): Containing an artist_url dict, and a tokens dict Returns: A dictionary containing the updated artist_url, the social_profile_id related to it and an error if one occurred. """ artist_url = params['artist_url'] response_data = {'artist_url': artist_url} # Validate url platform_id = validate_url(artist_url['url'], artist_url['site_id']) if not platform_id: response_data['error'] = error.SYNC_ARTIST_INVALID_URL return response_data # Get recommended profile recommended_profile = get_recommended_profile( platform_id, artist_url['site_id'], params['tokens']) if not recommended_profile: response_data['error'] = error.SYNC_ARTIST_NO_PROFILE return response_data # Search if a social_profile exists for the recommended profile. found_profile_response = ( social_profile.get_social_profile_with_platform_id_and_platform( recommended_profile['platform_id'], recommended_profile['platform'])) if found_profile_response: profile = found_profile_response.message else: profile = create_social_profile(recommended_profile) if not profile: response_data['error'] = error.SYNC_ARTIST_SOCIAL_PROFILE_CREATE return response_data response_data['social_profile'] = profile artist_url['social_profile_id'] = profile['social_profile_id'] artist_url['evaluated_for_collection'] = True update_url_response = artist_url_model.update_artist_url(artist_url) if not update_url_response: response_data['error'] = error.SYNC_ARTIST_URL_UPDATE return response_data response_data['artist_url'] = update_url_response.message return response_data def get_recommended_profile(platform_id, site_id, tokens): """Get recommended profile. Args: platform_id (str): The platform id. site_id (int): The site id. tokens (dict): A dict containing the access tokens for the social platforms. Returns: A dict containing the recommended profile if one was found or None. """ # Get recommended profile recommended_profile = None # Facebok if site_id == 2: token = tokens['facebook'] recommended_profile = facebook_collector.recommend_facebook_profile( platform_id, token) # Instagram if site_id == 46: token = tokens['instagram'] recommended_profile = instagram_collector.recommend_instagram_profile( platform_id, token) # Spotify if site_id == 47: token = tokens['spotify'] recommended_profile = spotify_collector.recommend_spotify_profile( platform_id, token) return recommended_profile def create_social_profile(recommended_profile): """Create a social profile from the recommended profile. Created_by field will have value 'jenkins' to distinguish that it was created by the sync jenkins job. Args: recommended_profile (dict): The recommended profile data. """ required_fields = ['platform', 'platform_id', 'platform_name'] if not recommended_profile: return if not all(field in recommended_profile for field in required_fields): return data = { 'platform': recommended_profile['platform'], 'platform_id': recommended_profile['platform_id'], 'platform_name': recommended_profile['platform_name'], 'collection_scheduled_time': datetime.now(), 'created_by': 'jenkins' } created_profile_response = social_profile.create_social_profile(data) if created_profile_response: return created_profile_response.message def validate_url(url, site_id): """Validate a platform url. Args: url (str): The url. site_id (int): The platform id 2 - Facebook, 46 - Instagram, 47 - Spotify Returns: The extracted platform id or None if the url is invalid """ platform_id = None # Facebook if site_id == 2: platform_id = match_facebook_url_patterns(url) if platform_id and len(platform_id) < 4: platform_id = None return platform_id def match_facebook_url_patterns(url): """Match facebook urls. Args: url (str): The url to be checked for validity as a facebook url. Returns: The platform_id or None if the url is not valid. """ platform_id = None # Try to match 'profile.php?id=576713340' urls. url_regex = re.match('profile.php\?id=([0-9]+)', url) if url_regex and url_regex.group(0): found_match = url_regex.group(0) platform_id = found_match[found_match.index('id=') + len('id='):] # Try to match 'pages/Test-Artist/175587235021' urls. url_regex = re.match('.*pages/.*/([0-9]+)', url) if url_regex and url_regex.group(0): found_match = url_regex.group(0) platform_id = found_match[found_match.rfind('/') + 1:] # Try to match 'Artist-Name-123456' url_regex = re.match('.*-[0-9]+', url) if url_regex and url_regex.group(0): found_match = url_regex.group(0) platform_id = found_match[found_match.rfind('-') + 1:] return platform_id