"""Implementation of bulk report trigger.""" import os from itertools import count from lambdacommon.common_config import logger from . import config, constants from .schemas import CoveredTrackKey, SplitRow from .split_levels import SplitLevel, SubaccountSplitLevel, TrackSplitLevel from .utils import csv, rds, snowflake # Define the hierarchy of split levels to process, from most granular to least granular SPLIT_LEVEL_HIERARCHY: list[SplitLevel] = [ TrackSplitLevel(), SubaccountSplitLevel(), ] def run(report_run_uuid: str, skip_update: bool = False): try: trigger_report_generation(report_run_uuid, skip_update) except Exception: if not skip_update: rds.update_reports_status(report_run_uuid, constants.REPORT_STATUS_ERROR) raise finally: snowflake.close_connection() rds.close_connection() def trigger_report_generation(report_run_uuid: str, skip_update: bool): """Trigger the report generation. Args: report_run_uuid (str): UUID of the report run to generate """ logger.info(f"Getting period IDs for report run with UUID: {report_run_uuid}") period_ids, trigger_type = rds.get_report_run(report_run_uuid) invalid_collaborators = [] if trigger_type == constants.TRIGGER_TYPE_AUTO: invalid_collaborators = validate_splits_for_report_run(report_run_uuid) sync_splits(report_run_uuid, invalid_collaborators) sync_collaborators(report_run_uuid, invalid_collaborators) if len(invalid_collaborators) > 0 and not skip_update: logger.info("Setting invalid reports status to ERROR") rds.update_reports_status( report_run_uuid, constants.REPORT_STATUS_ERROR, with_collaborator_ids=invalid_collaborators, ) logger.info("Triggering async report generation query") reports_query_id = snowflake.trigger_reports_query_async( report_run_uuid, period_ids, ) if not skip_update: logger.info(f"Updating report run with snowflake query ID: {reports_query_id}") rds.update_snowflake_query_id(report_run_uuid, reports_query_id) def validate_splits_for_report_run(report_run_uuid: str) -> list: invalid_collaborators = set() if failed := rds.get_collaborators_with_any_gross_split(report_run_uuid): invalid_collaborators.update(failed) logger.error("Report run includes gross splits") if failed := rds.get_collaborators_with_any_split_exceeding_100pc(report_run_uuid): invalid_collaborators.update(failed) logger.error("Report run includes any one split exceeding 100%") if failed := rds.get_collaborators_with_combined_track_splits_exceeding_100pc(report_run_uuid): invalid_collaborators.update(failed) logger.error("Report run includes combined track splits exceeding 100%") return list(invalid_collaborators) def sync_splits(report_run_uuid: str, invalid_collaborators: list): """Sync splits to Snowflake.""" logger.info("Getting splits from RDS") all_splits: list[SplitRow] = [] tracks_covered: set[CoveredTrackKey] = set() synthetic_ids = count(-1, -1) # Process splits level by level, starting from the most granular. # This ensures that if a track is covered by a split at a more granular # level, it will be excluded from splits at less granular levels. E.g. if # a subaccount split covers a number of tracks, if any of those tracks # have splits at the track level those track-level splits would be used # and the subaccount split would be ignored. for level in SPLIT_LEVEL_HIERARCHY: splits_at_level = level.get_splits(report_run_uuid, invalid_collaborators) track_splits_from_level, newly_covered = level.to_track_splits( splits_at_level, tracks_covered, synthetic_ids ) all_splits.extend(track_splits_from_level) tracks_covered = newly_covered logger.info("Writing splits to temporary file") temp_file_path = csv.write_temporary_csv(all_splits) logger.info(f"Temporary file path: {temp_file_path}") logger.info("Creating temporary splits table") snowflake.create_splits_table() logger.info("Uploading splits file to internal stage") snowflake.upload_file_to_table_stage(temp_file_path, config.SNOWFLAKE_OBJECTS["temp_split"]) logger.info("Copying splits data to table") snowflake.copy_into_table_from_stage(config.SNOWFLAKE_OBJECTS["temp_split"]) logger.info("Deleting temporary splits file") os.remove(temp_file_path) def sync_collaborators(report_run_uuid: str, invalid_collaborators: list): """Sync collaborators to Snowflake.""" logger.info("Creating temporary collaborators table") snowflake.create_collaborators_table() logger.info("Getting collaborators from RDS") collaborators = rds.get_collaborators_for_report_run(report_run_uuid, invalid_collaborators) logger.info("Writing collaborators to temporary file") temp_file_path = csv.write_temporary_csv(collaborators) logger.info(f"Temporary file path: {temp_file_path}") logger.info("Uploading collaborators file to internal stage") snowflake.upload_file_to_table_stage( temp_file_path, config.SNOWFLAKE_OBJECTS["temp_collaborator"] ) logger.info("Copying collaborators data to table") snowflake.copy_into_table_from_stage(config.SNOWFLAKE_OBJECTS["temp_collaborator"]) logger.info("Deleting temporary collaborators file") os.remove(temp_file_path)