"""Tasks for generate reserve payouts based on Accounting data.""" import functools import json import logging import time from garcon import task from accounting.flows.reserve_payouts import setting from accounting.flows.reserve_payouts.connectors import dynamodb from accounting.flows.reserve_payouts.connectors import mysql from accounting.flows.reserve_payouts.connectors import ows_manual_adjustment from accounting.flows.reserve_payouts.connectors import sns from accounting.flows.reserve_payouts.connectors import sqs from accounting.flows.reserve_payouts.constants import ( dynamodb as dynamodb_constants) from accounting.flows.reserve_payouts.constants import ( ows_manual_adjustment as ows_manual_adjustment_constants) from accounting.flows.reserve_payouts.constants import ( task as task_constants) def notify_status(status, task, message): """Notify the administrator that flow/task has failed. Args: status (str): 'error' or 'success' status task (str): task name message (str): message to include in notification """ status_message = '{task}: {message}'.format(task=task, message=message) sns.send_status_message(status, status_message) def get_vendor_transactions_and_contract_terms(period_id): """Query the database and process the results. Args: period_id (int): period identifier. Returns: dict: with phys transactions sum and reserve payout contract terms for labels. """ physical_transactions = mysql.get_physical_transactions_sum(period_id) label_data = mysql.vendor_query_result_to_dict(physical_transactions) if not label_data: return {} contract_details = mysql.get_vendor_contracts( period_id, label_data.keys()) contract_details_dict = mysql.vendor_query_result_to_dict(contract_details) for vendor_id, data in label_data.items(): vendor_contract_details = contract_details_dict[vendor_id] data.update(vendor_contract_details) return label_data def calculate_reserve_and_payout_values(period_id, data): """Calculate all the reserves and payouts for one vendor. Args: period_id (int): period identifier. data (dict): vendor related data that is needed for the calculations. Returns: list: list of dicts with calculated reserves and payouts suitable for POST calls to ows-manual-adjustment micro-service. """ vendor_id = data['vendor_id'] amount_in_original_currency = data['amount_in_original_currency'] currencies_id = data['currencies_id'] exchange_rate = data['exchange_rate'] reserve_rate = data['reserve_rate'] if reserve_rate == 0: return [] number_of_month_before_payout = data['number_of_months_before_payout'] number_of_installments = data['number_of_installments'] if number_of_installments == 0: return [] reserve_in_orig_curr = amount_in_original_currency * reserve_rate reserve_in_usd = reserve_in_orig_curr * exchange_rate installment_in_orig_curr = reserve_in_orig_curr / number_of_installments installment_in_usd = installment_in_orig_curr * exchange_rate first_payout_period = period_id + number_of_month_before_payout last_payout_period = first_payout_period + number_of_installments adjustments = [{ 'amount': -reserve_in_usd, 'amount_in_original_currency': -reserve_in_orig_curr, 'category_id': ows_manual_adjustment_constants.RESERVE_HELD, 'adjust_for_period_id': period_id, 'apply_to_period_id': period_id, 'parent_id': vendor_id, 'currencies_id': currencies_id, }] for apply_to_period_id in range(first_payout_period, last_payout_period): payout_adjustment = { 'amount': installment_in_usd, 'amount_in_original_currency': installment_in_orig_curr, 'category_id': ows_manual_adjustment_constants.RESERVE_RELEASED, 'adjust_for_period_id': period_id, 'apply_to_period_id': apply_to_period_id, 'parent_id': vendor_id, 'currencies_id': currencies_id, } adjustments.append(payout_adjustment) return adjustments @task.decorate(timeout=300) def bootstrap( activity, period_id, skip_prepare_label_data=False, validate_only=False): """Make all proper steps for reserve_payouts workflow. Args: activity (ActivityWorker): the activity worker. period_id (int): period id value, e.g 214 skip_prepare_label_data (bool): indicates if the prepare_label_data task should be skipped validate_only (bool): indicates if only the validation should be performed. If the flag is set it will indicate that the flow should skip other steps except for the validation step Returns: dict: context variables to be passed along in the workflow """ assert period_id, 'Missing: period_id' response = { 'period_id': int(period_id), 'skip_prepare_label_data': skip_prepare_label_data, 'validate_only': validate_only, } connectors = (dynamodb, ows_manual_adjustment, sqs) health_checks = [getattr(connector, 'health_check') for connector in connectors] for db_name in setting.DATABASES: mysql_health_check = functools.partial(mysql.health_check, db_name) health_checks.append(mysql_health_check) for health_check in health_checks: result = health_check() if result.bool: continue # something went wrong during the health check. notify_status('error', 'bootstrap', result.message) return {'stop': True, 'error_message': result.message} return response def check_dynamodb_sqs_item_count(task_name): """Ensure that the amount of DynamoDB and SQS items matches. Args: task_name (str): name of the task that performs the check Returns: dict: result of the check, dict will contain 'dynamodb_count' and 'sqs_count'. If the check was not successful it will also contain 'stop' equal to True. """ table_item_count = dynamodb.get_table_items_count() queue_message_count = sqs.get_queue_message_count() result = { 'dynamodb_count': table_item_count, 'sqs_count': queue_message_count} if table_item_count != queue_message_count: msg = ( 'DynamoDB item count and SQS message count mismatch, ' 'DynamoDB: {dynamodb_count} SQS: {sqs_count}') notify_status('error', task_name, msg.format(**result)) result['stop'] = True return result @task.decorate(timeout=2100) def prepare_temp_table(activity, period_id): """Truncate the temp aggregation table. Args: activity (ActivityWorker): the activity worker. period_id (int): period id value, e.g 214 Returns: dict: the result of task execution, contains affected rowcount """ logging.info('Truncating temporary Table...') mysql.truncate_reserves_temp_table() logging.info('Inserting data to the temporary table...') rowcount = mysql.populate_reserves_temp_table(period_id) logging.info('Done. Affected row count: {}'.format(rowcount)) result = { 'stop': rowcount == 0, 'rowcount': rowcount, } return result @task.decorate(timeout=7000) def prepare_label_data(activity, period_id): """Extract the data from the database and populate DynamoDB and SQS. Args: activity (ActivityWorker): the activity worker. period_id (int): period id value, e.g 214 """ vendor_data = get_vendor_transactions_and_contract_terms(period_id) item_count = len(vendor_data) if item_count == 0: return {'stop': True, 'reason': 'No data.'} dynamodb.batch_write_vendor_data(vendor_data) message_list = [sqs.construct_vendor_message(label_data) for label_data in vendor_data.values()] sqs.batch_vendor_send_messages(message_list) check_result = check_dynamodb_sqs_item_count('prepare_label_data') check_result['vendr_items_count'] = item_count return check_result @task.decorate(timeout=7000) def validate_reserve_payout_calculation(activity): """Validate the SQS and DynamoDB records after the calculation. Returns: dict: empty dict in case of success or dict with stop flag and validation_error_message """ sqs_validation = sqs.get_queue_message_count() == 0 dynamodb_validation = dynamodb.validate_successful_items_count() if sqs_validation and dynamodb_validation: notify_status( 'success', 'validate_reserve_payout_calculation', task_constants.SUCCESS_FLOW_COMPLETION_MESSAGE) return {} msg = 'Validation failure, {}.' if not sqs_validation: msg = msg.format('not all SQS messages were consumed') else: msg = msg.format('DynamoDB items have other than SUCCESS status') notify_status('error', 'validate_reserve_payout_calculation', msg) return {'stop': True, 'validation_error_message': msg} def process_reserve_payout(period_id, data): """Process reserve and payouts for vendor. Args: period_id (int): period identifier data (dict): vendor data Returns: tuple: total number of adjustments, list of dicts with failure details. """ manual_adjustments = calculate_reserve_and_payout_values(period_id, data) failed_adjustments = [] for manual_adjustment in manual_adjustments: response = ows_manual_adjustment.post_manual_adjustment( manual_adjustment) time.sleep(0.2) if response.status_code in (200, 201): continue category_id = manual_adjustment['category_id'] if category_id == ows_manual_adjustment_constants.RESERVE_HELD: category = 'Reserve held' else: category = 'Reserve released' failed_adjustments.append({ 'category': category, 'manual_adjustment': manual_adjustment, 'response_status_code': response.status_code, 'response_text': response.text, }) return len(manual_adjustments), failed_adjustments def get_calculation_failure_msg( vendor_id, total_adjustments, failed_adjustments): """Construct failure message for vendor. Args: vendor_id (int): label id total_adjustments (int): the total number of manual adjustments for vendor failed_adjustments (list): list of failed adjustments Returns: str: failure message """ failed_categories = [] for item in failed_adjustments: category = item['category'] if category not in failed_categories: failed_categories.append(category) failed_categories_str = ', '.join(failed_categories) msg = task_constants.VENDOR_CALCULATION_FAIL_MSG_TEMPLATE.format( vendor_id=vendor_id, failed_categories=failed_categories_str, num_failed=len(failed_adjustments), num_total=total_adjustments) return msg @task.decorate(timeout=7000) def calculate_reserve_payouts(activity, period_id): """Calculate the reserves and payouts using prepared data. Args: activity (ActivityWorker): the activity worker. period_id (int): period identifier. Returns: dict: task execution details """ message = sqs.get_message() failed_adjustments = [] failed_adjustments_details = [] while message: data = json.loads(message.body) vendor_id = data['vendor_id'] dynamodb.update_vendor_status(vendor_id, dynamodb_constants.PROCESSING) total_adjustments, vendor_failed_adjustments = process_reserve_payout( period_id, data) status = dynamodb_constants.SUCCESS if vendor_failed_adjustments: msg = get_calculation_failure_msg( vendor_id, total_adjustments, vendor_failed_adjustments) failed_adjustments.append(msg) failed_adjustments_details.extend(vendor_failed_adjustments) status = dynamodb_constants.ERROR dynamodb.update_vendor_status(vendor_id, status) message.delete() message = sqs.get_message() if failed_adjustments: msg = '\n'.join(failed_adjustments) notify_status('error', 'calculate_reserve_payouts', msg) response = { 'failed_adjustments': failed_adjustments, 'failed_adjustments_details': failed_adjustments_details, } else: response = {} return response