"""Module that creates .csv file and saves product data into it.""" from collections import OrderedDict import copy import csv from datetime import datetime import tempfile from oto import response from label_copy_export.connectors import loggly from label_copy_export.connectors import s3 from label_copy_export.connectors import sentry from label_copy_export.constants import csv as csv_constants from label_copy_export.constants import error from label_copy_export.constants import lce_history from label_copy_export.constants import ows_services from label_copy_export.models import history from label_copy_export.models import label_copy_export from label_copy_export.models import ows_artist from label_copy_export.models import ows_assets from label_copy_export.models import ows_pricing from label_copy_export.models import ows_project_manager from label_copy_export.models import ows_product_physical from label_copy_export.utils import file_utils from label_copy_export.validation.validate_csv import validate_sqs_message logger = loggly.get_current_logger() PHYSICAL_MUSIC_PRICING_FAMILY = { 'name': 'Physical Music', 'pricing_family_id': 4 } def generate_label_copy_export_filename(project_id): """Generate filename for CSV file. This function generates filename in format YYYYMMDD_{project_id}.csv Args: project_id (str): project id retrieved from message. Returns: str: generated filename for label copy export. """ files_timestamp = datetime.today().strftime( csv_constants.CSV_FILENAME_TIMESTAMP_FORMAT) output_file_name = ( '{timestamp}_{project_id}.csv' .format( timestamp=files_timestamp, project_id=project_id )) return output_file_name def normalize_csv_row_sequence(data_to_normalize, mapping_to_use): """Normalize product data according to csv columns sequence. Args: data_to_normalize (dict): product data to normalize. mapping_to_use (dict): mapping to use for normalizing data. Returns: dict: product data values to save as csv row. """ return { mapping_to_use[k]: v for k, v in data_to_normalize.items() if k in mapping_to_use} def get_project_data_from_microservices(project_id): """Collect project data for csv generation from different microservices. Args: project_id (str): project id to collect data for. Returns: dict: collected data. """ data = {} project_data = ows_project_manager.get_project(project_id) if not project_data: return project_data for data_key in [ ows_services.PROJECT_NAME, ows_services.PROJECT_CODE, ows_services.DESCRIPTION, ows_services.PROJECT_HIGHLIGHTS]: data[data_key] = project_data.message.get(data_key, '') project_artist = ows_artist.get_ows_artist( project_data.message[ows_services.ARTIST_ID]) if project_artist: data[ows_services.PROJECT_ARTIST] = project_artist.message[ ows_services.NAME] else: logger.info( 'There was no artist found for project {project_id}'.format( project_id=project_id)) return response.Response(message=data) def get_product_data_from_microservices(product_id, orchard_user_id=None): """Collect product data for csv generation from different microservices. Args: product_id (str): product id to collect data for. orchard_user_id (str): unique identifier of user, example: 'alw:25824'. Returns: Response: list of collected data. """ product_data_for_csv_generation = [] data = {} product_data = ows_product_physical.get_product_physical_data(product_id) if not product_data: return product_data for data_key in ows_services.PRODUCT_DATA_KEYS: data[data_key] = product_data.message.get(data_key, '') for field in ['wholesale_price', 'pricing']: data.pop(field) orchard_pricing_tier = get_orchard_pricing_tier_name( product_id, PHYSICAL_MUSIC_PRICING_FAMILY['pricing_family_id']) if orchard_pricing_tier: data['orchard_pricing_tier'] = orchard_pricing_tier.message data['release_status'] = csv_constants.RELEASE_STATUSES_MAPPING[ data['release_status']] image_path = ows_assets.get_large_cover_image_url(product_id) if image_path: data[ows_services.IMAGE_PATH] = image_path.message all_genres = ows_project_manager.get_project_genres() if not all_genres: return all_genres for item in all_genres.message: if item['id'] == product_data.message[ows_services.GENRE_ID]: data[ows_services.GENRE_NAME] = item['name'] break subgenres_for_genre = ows_project_manager.get_project_subgenres( product_data.message[ows_services.GENRE_ID]) if not subgenres_for_genre: return subgenres_for_genre for item in subgenres_for_genre.message: if item['id'] == product_data.message[ows_services.SUBGENRE_ID]: data[ows_services.SUBGENRE_NAME] = item['name'] break distribution_format_name = label_copy_export.get_distribution_format_name( product_id) if not distribution_format_name: return distribution_format_name else: data[ows_services.DISTRIBUTION_FORMAT_NAME] = ( distribution_format_name.message[ ows_services.DISTRIBUTION_FORMAT_NAME]) all_packaging_options = ows_product_physical.get_packaging_options() if not all_packaging_options: return all_packaging_options for item in all_packaging_options.message: if item['id'] == product_data.message[ows_services.PACKAGING_ID]: data[ows_services.PACKAGING] = item['name'] break product_tracks = ows_product_physical.get_product_physical_tracks( product_id) if not product_tracks: return product_tracks if product_tracks.message['items']: for track in product_tracks.message['items']: track[ows_services.PERFORMER] = ', '.join( track[ows_services.PERFORMER]) row = copy.deepcopy(data) row.update(track) product_data_for_csv_generation.append(row) else: product_data_for_csv_generation.append(data) return response.Response(message=product_data_for_csv_generation) def generate_csv(project_id, orchard_user_id=None): """Generate csv for given project id. Args: project_id (str): project id to generate csv for. orchard_user_id (str): unique identifier of user, example: 'alw:25824'. Returns: Response: generated csv file name. The successful Response contains the name of generated csv file in Response.message. The error Response returns the name of csv file in Response.errors['message']['csv_file']. """ project_data = get_project_data_from_microservices(project_id) if not project_data: return project_data project_products = ows_project_manager.get_products_for_project(project_id) if not project_products: return project_products temp_csv_file = tempfile.NamedTemporaryFile( mode='w', suffix='.csv', delete=False) csv_header_data_mapping_copy = copy.deepcopy( csv_constants.CSV_HEADER_DATA_MAPPING) csv_header_data_mapping_copy.pop('wholesale_price') csv_header_data_mapping_copy = OrderedDict( ('orchard_pricing_tier', 'Orchard Pricing Tier') if key == 'pricing' else (key, value) for key, value in csv_header_data_mapping_copy.items()) with temp_csv_file as csvfile: fieldnames = csv_header_data_mapping_copy.values() try: writer = csv.DictWriter( csvfile, fieldnames=fieldnames, dialect='excel') writer.writeheader() no_phys_products_for_project = True for product in project_products.message[csv_constants.ITEMS]: if product[csv_constants.DISTRIBUTION_FORMAT][ csv_constants.CONTEXT] == csv_constants.PHYSICAL: product_data = get_product_data_from_microservices( product[csv_constants.PRODUCT_ID], orchard_user_id) if not product_data: return response.create_error_response( code=error.ERROR_GENERATING_CSV, message={ 'exception': product_data.errors, 'csv_file': temp_csv_file.name}) for product_data_row in product_data.message: csv_row_to_write = product_data_row csv_row_to_write.update(project_data.message) normalized_csv_row_data = normalize_csv_row_sequence( csv_row_to_write, csv_header_data_mapping_copy) writer.writerow(normalized_csv_row_data) no_phys_products_for_project = False if no_phys_products_for_project: normalized_csv_row_data = normalize_csv_row_sequence( project_data.message, csv_header_data_mapping_copy) writer.writerow(normalized_csv_row_data) except csv.Error as ex: if sentry.sentry_client: sentry.sentry_client.captureException() logger.error( 'Error while writing csv row for project {project_id}: ' '{ex}.'.format(project_id=project_id, ex=ex)) return response.create_error_response( code=error.ERROR_GENERATING_CSV, message={'exception': ex.args, 'csv_file': temp_csv_file.name}) return response.Response(message=temp_csv_file.name) def csv_processing(message): """Generate csv for given project id, save to s3 and remove. Args: message (JSONMessageExt): message pulled from SQS-message. Returns: Response: successful response in case there was no exceptions. """ validated_message = validate_sqs_message(message) if not validated_message: return response.create_error_response( code=error.ERROR_CODE_CSV_GENERATION, message=error.ERROR_MESSAGE_INVALID_SQS) project_id = message.get_body()['project_id'] job_id = message.get_body()['job_id'] orchard_user_id = message.feature_flag_user_context set_job_in_progress = history.change_status( job_id, lce_history.IN_PROGRESS) if not set_job_in_progress: return set_job_in_progress generated_csv_file = generate_csv(project_id, orchard_user_id) if not generated_csv_file: generation_error_message = generated_csv_file.errors['message'] if generation_error_message and isinstance( generation_error_message, dict): file_utils.remove_file(generation_error_message.get('csv_file')) return _handle_csv_generation_error_response( job_id, generated_csv_file) file_path = generated_csv_file.message s3_key_name = generate_label_copy_export_filename(project_id) uploading_result = s3.upload_file_to_s3(file_path, s3_key_name) file_utils.remove_file(generated_csv_file.message) if not uploading_result: return _handle_csv_generation_error_response(job_id, uploading_result) set_job_completed = history.change_status(job_id, lce_history.COMPLETED) if not set_job_completed: return set_job_completed return response.Response() def _handle_csv_generation_error_response( job_id, csv_generation_and_uploading_result): """Handle .csv generation error response. Args: job_id (int): job id. csv_generation_and_uploading_result (Response): failure .csv generation and uploading response. Returns: Response: error response with appropriate message. """ set_job_error = history.change_status(job_id, lce_history.ERROR) if not set_job_error: return set_job_error return response.create_error_response( code=error.ERROR_GENERATING_CSV, message=csv_generation_and_uploading_result.errors['message']) def get_orchard_pricing_tier_name(product_id, pricing_family_id): """Get Orchard Pricing Tier name for product. Args: product_id (int): id of the product. pricing_family_id (int): id of pricing family for which we get the list of Orchard Pricing Tiers. Returns: Response: message contains name of Orchard Pricing Tier or error response. """ pricing_tiers_for_family_response = ows_pricing.get_orchard_pricing_tiers( pricing_family_id) if not pricing_tiers_for_family_response: return pricing_tiers_for_family_response pricing_tiers_for_family = pricing_tiers_for_family_response.message pricing_tier_for_product_response = ( ows_pricing.get_orchard_pricing_tier_for_product( product_id, pricing_family_id)) if not pricing_tier_for_product_response: return pricing_tier_for_product_response product_pricing_tier_id = pricing_tier_for_product_response.message[ ows_services.ORCHARD_PRICING_TIER_ID] product_pricing_name = next(( tier[ows_services.NAME] for tier in pricing_tiers_for_family if tier[ows_services.ORCHARD_PRICING_TIER_ID] == product_pricing_tier_id), None) if not product_pricing_name: return response.create_not_found_response( message=error.ERROR_ORCHARD_PROCING_TIER_NOT_FOUND_MESSAGE) return response.Response(message=product_pricing_name)