import os from datetime import date from shutil import copyfile from uuid import uuid4 from .config import OUTPUT_BUCKET from .constants import REPORT_STATUS_ERROR, TRANSACTION_TYPE_REVENUE, TRIGGER_TYPE_AUTO from .utils import s3 from .utils.db import ( clear_report_contract, create_transactions, get_open_statement_period_ids_for_vendors, get_report_info_by_collaborator_id, get_report_run_info, update_report, update_reports_status, ) from .utils.query_result import get_query_result_rows, has_field_value from .utils.report import ReportWriter, create_report, write_row from .utils.report_writer import Report from .utils.ses import send_notification_email from .utils.transaction import Transaction, get_transaction_description def process( bucket, key, skip_update=False, skip_upload=False, skip_notification=False, skip_add_to_royalties=False, ): try: generate_reports( bucket, key, skip_update, skip_upload, skip_notification, skip_add_to_royalties, ) except Exception as e: report_run_uuid = key.split(".")[0] update_reports_status(report_run_uuid, REPORT_STATUS_ERROR) raise e def generate_reports( bucket, key, skip_update, skip_upload, skip_notification, skip_add_to_royalties ): report_run_uuid = key.split(".")[0] report_run = get_report_run_info(report_run_uuid) report_info_by_collaborator_id = get_report_info_by_collaborator_id(report_run_uuid) if not skip_update: clear_report_contract(report_run_uuid) reports = [] reports.extend( create_reports_from_query_results( bucket=bucket, key=key, report_run_uuid=report_run_uuid, report_run_name=report_run["name"], number_format=report_run["number_format"], period_name=report_run["period_name"], report_info_by_collaborator_id=report_info_by_collaborator_id, skip_upload=skip_upload, skip_update=skip_update, ) ) reports.extend( create_empty_reports( report_run_uuid=report_run_uuid, report_run_name=report_run["name"], number_format=report_run["number_format"], period_name=report_run["period_name"], report_info_by_collaborator_id=report_info_by_collaborator_id, skip_upload=skip_upload, skip_update=skip_update, ) ) if skip_add_to_royalties or report_run["trigger_type"] != TRIGGER_TYPE_AUTO: print("Skipping adding reports to royalties") else: add_reports_to_royalties(reports, report_run) if skip_notification or not report_run["notification_email"]: print(f"Skipping notification email for {report_run['name']}") else: send_notification_email(report_run["notification_email"], report_run["name"]) def finalise_report(writer: ReportWriter, skip_upload: bool, skip_update: bool): writer.finalise() output_path = "/".join( [ writer.report.report_run_uuid, str(writer.report.report_id), f"{writer.report.filename}.{writer.extension}", ] ) if skip_upload: print(f"Skipping upload of report for collaborator {writer.report.collaborator_id}") local_path = f"output/{output_path}" print(f"Writing to {local_path}") os.makedirs(os.path.dirname(local_path), exist_ok=True) copyfile(writer.output_filename, local_path) else: s3.upload_file(writer.output_filename, OUTPUT_BUCKET, output_path) # Delete the temporary file os.remove(writer.output_filename) if skip_update: print(f"Skipping update of report for collaborator {writer.report.collaborator_id}") else: update_report(writer, output_path) def create_reports_from_query_results( bucket: str, key: str, report_run_uuid: str, report_run_name: str, number_format: str, period_name: str, report_info_by_collaborator_id, skip_upload: bool, skip_update: bool, ) -> list[Report]: """Create reports for collaborators with transactions in query results.""" query_result_file = s3.get_object(bucket, key) reports = [] writer = None for row in get_query_result_rows(query_result_file): # If this is the whole report run total row skip it if not has_field_value(row, "collaboratorid"): continue # If this is a collaborator total row if not has_field_value(row, "txn_id") and not has_field_value(row, "contract_id"): # If there is an existing report finalise it first if writer: finalise_report(writer, skip_upload, skip_update) report_info = report_info_by_collaborator_id.pop(row["collaboratorid"], None) # If a report should be created if report_info and report_info["status"] != REPORT_STATUS_ERROR: # Create a new writer writer = create_report( report_id=report_info["id"], collaborator_id=row["collaboratorid"], total=row["collaborator_share"], currency=( row["account_payee_currency"] if row["account_payee_currency"] != "\\N" else report_info["currency"] ), collaborator_name=report_info["collaborator_name"], collaborator_internal_id=report_info["collaborator_internal_id"], filename=report_info["filename"], report_run_name=report_run_name, period_name=period_name, vendor_id=report_info["vendor_id"], report_run_uuid=report_run_uuid, number_format=number_format, ) reports.append(writer.report) else: # Otherwise clear the writer writer = None # This really just to satisfy typing - it should always be true since the results always # start with a collaborator transaction row elif writer: # If this is a contract total row if not has_field_value(row, "txn_id"): # "0" represents no contract if row["contract_id"] != "0": writer.report.contract_totals[row["contract_id"]] = row["collaborator_share"] else: # Otherwise this is a transaction row so we should write it write_row(writer, row) # Close and upload the final output file if writer: finalise_report(writer, skip_upload, skip_update) return reports def create_empty_reports( report_run_uuid: str, report_run_name: str, number_format: str, period_name: str, report_info_by_collaborator_id, skip_upload: bool, skip_update: bool, ) -> list[Report]: """Create zero value reports for collaborators without transactions in query results file.""" reports = [] for collaborator_id, report_info in report_info_by_collaborator_id.items(): if report_info.get("status") == REPORT_STATUS_ERROR: continue writer = create_report( report_id=report_info["id"], collaborator_id=collaborator_id, total="0", currency=report_info["currency"], collaborator_name=report_info["collaborator_name"], collaborator_internal_id=report_info["collaborator_internal_id"], filename=report_info["filename"], report_run_name=report_run_name, period_name=period_name, vendor_id=report_info["vendor_id"], report_run_uuid=report_run_uuid, number_format=number_format, ) finalise_report(writer, skip_upload, skip_update) reports.append(writer.report) return reports def add_reports_to_royalties(reports: list[Report], report_run): """ Add all the reports in a given report_run_uuid to royalties. """ if not reports: print("No reports to add to royalties") return vendor_ids = list(set(report.vendor_id for report in reports)) statement_period_ids = get_open_statement_period_ids_for_vendors(vendor_ids) creation_batch_uuid = str(uuid4()) date_today = date.today().strftime("%Y-%m-%d") transactions = [] for report in reports: statement_period_id = statement_period_ids.get(report.vendor_id) description = get_transaction_description(report, report_run, date_today) if not statement_period_id: raise ValueError(f"Statement period not found for vendor {report.vendor_id}") transactions.append( Transaction( collaborator_id=report.collaborator_id, report_id=report.report_id, statement_period_id=statement_period_id, type=TRANSACTION_TYPE_REVENUE, original_amount=report.total, chargeable_amount=report.total, currency=report.currency, date=date_today, description=description, creation_batch_uuid=creation_batch_uuid, ) ) # Create transactions in the database if transactions: create_transactions(transactions)