"""Generate payment summary report task.""" from math import ceil import pandas from lib.constants import PAYMENT_GROUP_PAYMENT_ACTIONS, PAYMENT_SCHEDULE_MAP from lib.constants import PAYMENT_SUMMARY_HEADERS_MAP from lib.constants import PAYMENT_SUMMARY_HEADERS_MAP_CALCULATE_PAYMENTS from lib.utils import ows from tasks.payments_generate_export import helpers def generate_payment_summary_export_task(dag_run: dict, **kwargs): """Task to generate a report of payment_group_payment_accounts to save on S3. Args: dag_run (dict): a dag's config """ event = helpers.get_event_from_params(dag_run, **kwargs) payment_group_payment = ows.get_payment_group_payment_details(event.target_id) headers_map = PAYMENT_SUMMARY_HEADERS_MAP try: # feature flag check using states helpers.get_abacus_state( PAYMENT_GROUP_PAYMENT_ACTIONS.CALCULATE_PAYMENTS, event.target_id ) headers_map = PAYMENT_SUMMARY_HEADERS_MAP_CALCULATE_PAYMENTS except ValueError: pass report_s3_path = helpers.build_report_summary_location( payment_group_payment.get('payment_group_payment_id'), payment_group_payment.get('payment_name') ) raw_report_data = _get_report_data(event.target_id) payment_group_data = pandas.DataFrame( pandas.Series( _get_payment_group_payment_details(payment_group_payment) ) ) filtered_report_data = pandas.DataFrame( raw_report_data, columns=headers_map.keys() ) filtered_report_data['percent_difference'] = \ filtered_report_data['percent_difference'].map( lambda a: f'{a}%' if pandas.notnull(a) else a) filtered_report_data.fillna(value='-', inplace=True) formatted_report = filtered_report_data.rename(columns=headers_map) blank_line = pandas.DataFrame() payment_group_data.to_csv(report_s3_path.url, mode='w', index=False, header=False) blank_line.to_csv(report_s3_path.url, mode='a', index=False, lineterminator='\n') formatted_report.to_csv(report_s3_path.url, mode='a', index=False, sep='\t') def _get_report_data(payment_group_payment_id: int) -> list: """Request payment_group_payment_accounts from ows-payment in batches. Args: payment_group_payment_id (int): ID of the parent payment_group_payment Returns: list of dicts containing payment_group_payment_account data. """ limit = 5000 offset = 0 sort_by = 'balance_after_tax' sort_order = 'desc' report_data = [] response = ows.get_payment_group_payment_accounts( payment_group_payment_id, limit, offset, sort_by, sort_order ) report_data.extend(response.get('items', [])) num_of_batches = ceil(response.get('total_count', 1) / limit) for batch_num in range(1, num_of_batches): offset = batch_num * limit response = ows.get_payment_group_payment_accounts( payment_group_payment_id, limit, offset, sort_by, sort_order ) report_data.extend(response.get('items', [])) return report_data def _get_payment_group_payment_details(payment_group_payment: dict) -> list: """Request payment-group from ows-payment and statement-period from ows-royalties. Args: payment_group_payment (dict): payment group payment details Returns: list containing payment group payment details. """ created_at = payment_group_payment.get('created_at') payment_group_id = payment_group_payment.get('payment_group_id') payment_name = payment_group_payment.get('payment_name') statement_period_id = payment_group_payment.get('statement_period_id') payment_group = ows.get_payment_group_details(payment_group_id) group_criteria = payment_group.get('group_criteria') group_name = payment_group.get('group_name') payment_schedules = group_criteria.get('payment_schedules') payment_entities_map = ows.get_reference_payment_entities() payment_entities = group_criteria.get('reference_payment_entities', []) account_id = group_criteria.get('account_id', None) if account_id: account_payment_term = ows.get_account_payment_term_by_account_id(account_id) payment_schedule = account_payment_term.get('payment_schedule') payment_entity_id = account_payment_term.get('payment_entity_id') payment_entities = [payment_entities_map[payment_entity_id]] payment_schedules = \ [PAYMENT_SCHEDULE_MAP[payment_schedule]] if payment_schedule else '-' else: if not payment_schedules: payment_schedules = ['All'] else: payment_schedules = [ PAYMENT_SCHEDULE_MAP[payment_schedule] for payment_schedule in payment_schedules ] payment_entities = [ payment_entities_map[payment_entity] for payment_entity in payment_entities ] statement_period = ows.get_statement_period_details(statement_period_id) statement_period_name = statement_period.get('statement_period_name', None) return [ payment_name, f'Statement Period {statement_period_name} ({statement_period_id})', created_at, group_name, ', '.join(payment_entities), ', '.join(payment_schedules) ]