"""Logic for metrics collection.""" import csv from datetime import datetime import json import os import time from uuid import uuid4 import boto3 from oto import response from oto import status from social_analytics import config from social_analytics.connectors import loggly from social_analytics.connectors import sentry from social_analytics.constants import collectors from social_analytics.constants import error from social_analytics.logic import facebook_collector from social_analytics.logic import instagram_collector from social_analytics.logic import spotify_collector PLATFORM_IDS = collectors.PLATFORM_AND_SOCIAL_METRIC_REFS['platforms'] METRIC_IDS = collectors.PLATFORM_AND_SOCIAL_METRIC_REFS['social_metrics'] logger = loggly.get_current_logger() def get_tokens(): """"Get social platform API tokens.""" facebook = '' instagram = '' spotify = '' if config.FACEBOOK_IS_ACTIVE: try: facebook = facebook_collector.refresh_token() except Exception as exc: logger.error( ('Exception occurred on refreshing facebook token {0}'.format( str(exc)))) if config.INSTAGRAM_IS_ACTIVE: try: instagram = instagram_collector.refresh_token() except Exception as exc: logger.error( ('Exception occurred on refreshing instagram token {0}'.format( str(exc)))) if config.SPOTIFY_IS_ACTIVE: try: spotify = spotify_collector.refresh_token()['access_token'] except Exception as exc: logger.error( ('Exception occurred on refreshing instagram token {0}'.format( str(exc)))) return { 'facebook': facebook, 'instagram': instagram, 'spotify': spotify } def collect(collection_params): """Collect metrics for a social profile. Args: collection_params (dict): a dictionary containing values required for fetching metrics e.g. { 'social_profile': { 'social_profile_id': 123, 'platform': 'facebook', 'platform_id': 345, }, 'tokens': { 'facebook': 123, 'instagram': 456, 'spotify': 789 } } Returns: a dictionary containing social profile id, platform and metrics fetched, or None if no platform matched. """ social_profile = collection_params['social_profile'] logger.info(('Collecting for {0} '.format(social_profile))) tokens = collection_params['tokens'] metrics_response = { 'social_profile_id': social_profile['social_profile_id'], 'platform': social_profile['platform'], 'data_response': '' } if social_profile['platform'] == 'facebook' and tokens['facebook']: metrics_response['data_response'] = ( facebook_collector .collect_facebook_metrics( social_profile['platform_id'], tokens['facebook']) ) elif social_profile['platform'] == 'instagram' and tokens['instagram']: metrics_response['data_response'] = ( instagram_collector .collect_instagram_metrics( social_profile['platform_id'], tokens['instagram']) ) elif social_profile['platform'] == 'spotify' and tokens['spotify']: metrics_response['data_response'] = ( spotify_collector .collect_spotify_metrics( social_profile['platform_id'], tokens['spotify']) ) logger.info(('The metrics response is: {0} '.format(metrics_response))) if metrics_response['data_response']: return metrics_response def upload_file_to_s3(file_path, remote_file_name): """Upload csv file to S3 bucket. Args: file_path (str): the path of the file to be uploaded to S3. remote_file_name (str): the name with which to create the remote file. Returns: response: The HTTP response received upon submission of the request. """ s3 = boto3.client( 's3', aws_access_key_id=config.AWS_ACCESS_KEY, aws_secret_access_key=config.AWS_SECRET_KEY) upload_error = None try: s3.upload_file(file_path, config.AWS_S3_BUCKET, remote_file_name) except Exception as e: upload_error = e finally: os.remove(file_path) if upload_error: return upload_error def generate_metrics_remote_file_name(): """Generate a name for the file to be uploaded to s3. Returns: remote_file_name (str): a file name following the format s3://bucket_name/year/month/day_of_month/hour/minute/uuid.csv e.g. s3://cucumbers/social-collection/2017/05/04/17/12/04/12345.csv """ date = datetime.now() remote_file_name = ( 'social-collection/{year}/{month}/{day}/{hour}/' '{minute}/{second}/{uuid}.csv' .format(year=date.year, month=date.month, day=date.day, hour=date.hour, minute=date.minute, second=date.second, uuid=uuid4()) ) return remote_file_name def generate_raw_data_remote_json_file_name(platform, date_as_timestamp): """Create remote json file path. Args: platform (str): the platfrom_name e.g. Facebook. date_as_timestamp (str): unix timestamp as string. Returns: a string with the remote json file path. """ remote_file_name = ( 'social-collection-raw-data/{platform}/{date_as_timestamp}/data.json' .format(platform=platform, date_as_timestamp=date_as_timestamp)) return remote_file_name def upload_metrics_json_to_s3(social_profiles): """Upload metrics json to s3. Args: social_profiles (list): the collected social profiles. Returns: An error if one occurred or None. """ # Save metrics to csv file file_path = convert_social_profiles_to_csv(social_profiles) # Upload csv file to s3 bucket remote_file_name = generate_metrics_remote_file_name() upload_error = upload_file_to_s3(file_path, remote_file_name) if upload_error: logger.error( ('Error uploading metrics to s3 {0}'.format(upload_error))) sentry_client = sentry.get_client() sentry_client.captureMessage( message='Errors while uploading csv data to s3', stack=True, extra={ 'message': 'Upload CSV file to s3 errors', 'errors': upload_error, 'status': 500}) return upload_error def upload_raw_json_to_s3(social_profiles): """Upload raw_json_to_s3. Args: social_profiles (list): the collected social profiles. Returns: A list of errors or None if no errors occurred. """ errors = [] platforms = [collectors.FACEBOOK, collectors.INSTAGRAM, collectors.SPOTIFY] date_as_timestamp = (str(int(time.time()))) for platform in platforms: raw_data = group_social_profiles_by_platform( platform, social_profiles) if not raw_data: continue file_path = create_raw_data_json_file(raw_data) remote_file_name = generate_raw_data_remote_json_file_name( platform, date_as_timestamp) upload_error = upload_file_to_s3(file_path, remote_file_name) if upload_error: logger.error( ('Error uploading raw data to s3 {0}'.format(upload_error))) sentry_client = sentry.get_client() sentry_client.captureMessage( message='Errors while uploading raw data to s3', stack=True, extra={ 'message': 'Upload raw data to s3 errors', 'errors': upload_error, 'status': 500}) errors.append(upload_error) return errors def create_raw_data_json_file(raw_data): """Create json file. Args: raw_data (list): list of dictionaries with the response from a social network. Returns: a string that represents the file name with the raw json """ file_name = 'raw_data.json' with open(file_name, 'w') as outfile: json.dump(raw_data, outfile) return file_name def group_social_profiles_by_platform(platform, social_profiles): """Group social profiles by a platform. Args: platform (str): The platform e.g. Facebook social_profiles (list): The collected social profiles Returns: a list with the social_profiles for a chosen platform. """ profiles = ( [social_profile['data_response'] for social_profile in social_profiles if social_profile and social_profile['platform'] == platform]) return profiles def convert_social_profiles_to_csv( social_profiles, path='./social_profiles.csv'): """Format the data from social profiles to csv. Args: social_profiles (list): A list with the data of all the collected social profiles. path (string): The temp path the csv would be saved. Returns: string with the path of the csv. """ with open(path, 'w') as csv_file: writer = csv.writer(csv_file) for social_profile in social_profiles: row = None try: logger.info( 'About to write row in csv for {0}'.format( social_profile['social_profile_id'])) row = process_response(social_profile) logger.info( 'Written row in csv for {0}'.format( social_profile['social_profile_id'])) except Exception as exc: logger.error( 'ERROR on metric csv creation for spID {0}'.format( social_profile['social_profile_id'])) logger.error(exc) sentry_client = sentry.get_client() sentry_client.captureMessage( message='Errors metrics csv creation', stack=True, extra={ 'message': ( 'ERROR on metric csv creation for ID: {0}'.format( social_profile['social_profile_id'])), 'errors': exc, 'status': 500}) pass if row: logger.info('About to write row in csv {}'.format(row)) writer.writerow(row) logger.info('Row {} has been written in csv'.format(row)) return path def process_response(social_profile): """Process Social Platform Response. Args: social_profile (dict): This dict contains platform, social_profile_id Returns: list with all the columns to create the csv row or None if there is an error from the social network response. """ metric = '' metric_value = None if 'error' in social_profile['data_response']: return None if social_profile['platform'] == 'spotify': metric = 'followers' metric_value = social_profile['data_response']['followers']['total'] elif social_profile['platform'] == 'facebook': metric = 'fan_count' metric_value = social_profile['data_response']['fan_count'] elif social_profile['platform'] == 'instagram': metric = 'followed_by' metric_value = ( social_profile['data_response']['data']['counts']['followed_by']) platform_id = next( p['platform_id'] for p in PLATFORM_IDS if p[ 'platform'] == social_profile['platform']) metric_id = next( m['metric_id'] for m in METRIC_IDS if m[ 'metric_name'] == metric) row = [social_profile['social_profile_id'], platform_id, '', metric_id, metric_value, str(datetime.now()), str(datetime.now())] return row def recommend_profile_for_social_network(profile_search_params): """Recommend or verify social profile. Args: profile_search_params (dict): A dictionary which holds the values for platform and platform name. e.g. : { 'platform': 'facebook' 'platform_name': 'Jeff Buckley' } { 'platform': 'facebook' 'platform_id': '123' } Returns: response.Response: containing the suggested social profile dict. """ platform_id = None platform_name = None if 'platform' not in profile_search_params: return response.create_error_response( code=status.BAD_REQUEST, message=error.ERROR_MESSAGE_BAD_PARAMS) else: platform = profile_search_params['platform'] if 'platform_id' in profile_search_params: platform_id = profile_search_params['platform_id'] if 'platform_name' in profile_search_params: platform_name = profile_search_params['platform_name'] if (not platform_name and not platform_id) or ( platform_name and platform_id): return response.create_error_response( code=status.BAD_REQUEST, message=error.ERROR_MESSAGE_BAD_PARAMS) if platform_name: return _handle_recommend_for_platform(platform_name, platform) if platform_id: return _handle_verify_for_platform(platform_id, platform) def _handle_recommend_for_platform(platform_name, platform): """Handle recommend for a social platform. Args: platform_name (str): The name of an artist in a social platform. platform (str): The actual platform e.g. facebook Returns: response.Response: containing the suggested social profile dict. """ recommendation = None if platform == collectors.FACEBOOK: recommendation = facebook_collector.recommend_facebook_profile( platform_name) if platform == collectors.INSTAGRAM: recommendation = instagram_collector.recommend_instagram_profile( platform_name, config.INSTAGRAM_ACCESS_TOKEN) if platform == collectors.SPOTIFY: recommendation = spotify_collector.recommend_spotify_profile( platform_name) if recommendation: return response.Response(recommendation) else: return response.create_not_found_response() def _handle_verify_for_platform(platform_id, platform): """Handle verify for a social platform. Args: platform_id (str): The id of an artist in a social platform. platform (str): The actual platform e.g. facebook Returns: response.Response: containing the suggested social profile dict. """ verified_profile = None if platform == collectors.FACEBOOK: verified_profile = facebook_collector.verify_facebook_profile( platform_id) if platform == collectors.INSTAGRAM: verified_profile = instagram_collector.verify_instagram_profile( platform_id, config.INSTAGRAM_ACCESS_TOKEN) if platform == collectors.SPOTIFY: verified_profile = spotify_collector.verify_spotify_profile( platform_id) if verified_profile: return response.Response(verified_profile) else: return response.create_not_found_response()