"""Grps Ingestor Flow Control.""" from time import sleep from common.aws import sfn from common.connectors.snowfalke_connector import execute_snowflake_query from common.constants.grps_ingestion_status import INGEST_STARTED import config from src.constants.queries import GET_UPC_TO_PROCESS, UPDATE_GRPS_INGESTION logger = config.app_logger def handler(event, context): """Lambda function to trigger grps ingestor sfn.""" logger.info( f'Beginning ingestion flow control. Event: {event} Context: {context}') max_sfn_count = event.get( 'MAX_STATE_MACHINES') or config.MAX_STATE_MACHINES # noqa # The delay between each Lambda batch invocation sfn_start_delay = event.get('SFN_START_DELAY') or config.SFN_START_DELAY # The number of executions to start for each sfn_start_delay sfn_batch_size = event.get('SFN_BATCH_SIZE') or config.SFN_BATCH_SIZE logger.info( f'MAX_STATE_MACHINES={max_sfn_count} ' f'SFN_START_DELAY={sfn_start_delay} ' f'SFN_BATCH_SIZE={sfn_batch_size}' ) current_sfn_count = sfn.get_full_execution_count_by_arn( config.GRPS_INGESTER_SFN_ARN, config.EXECUTION_RUNNING_STATUS ) nr_ingests = max_sfn_count - current_sfn_count if nr_ingests < 1: logger.info( f'Cannot run any more state machines. Currently running ' f'{current_sfn_count} with a limit of {max_sfn_count}' ) return logger.info( f'Currently running {current_sfn_count} executions with a limit of' f' {max_sfn_count}. Starting at most {nr_ingests} executions' ) upc_to_process = get_upc_to_process(nr_ingests) if not upc_to_process: logger.info('No deliveries to ingest. Ending ingestion flow control.') return i = 0 for upc, grps_ingestion_id, prod_no in upc_to_process: logger.info( f'Starting execution of ARN {config.GRPS_INGESTER_SFN_ARN}, ' f'with upc {upc}' ) response, sfn_name = sfn.start_execution_by_arn( config.GRPS_INGESTER_SFN_ARN, upc, grps_ingestion_id, prod_no ) update_grps_ingestion(grps_ingestion_id, INGEST_STARTED, sfn_name) logger.info(f'Updating row status for key {grps_ingestion_id}') if (i + 1) % sfn_batch_size == 0: # Sleeping to let sfn batch start. logger.info(f'Sleeping for {sfn_start_delay} milliseconds.') sleep(sfn_start_delay / 1000) i = i + 1 def get_upc_to_process(nr_ingests): """Get list upcs to process using sfn.""" upc_to_process = execute_snowflake_query(GET_UPC_TO_PROCESS, {'nr_ingests': nr_ingests}) return [(row['UPC'], row['GRPS_INGESTION_ID'], row['PROD_NO']) for row in upc_to_process] def update_grps_ingestion(grps_ingestion_id, status, sfn_name): """Update grps ingestion status.""" execute_snowflake_query(UPDATE_GRPS_INGESTION, {'status': status, 'sfn_url': f'{config.SFN_URL_PREFIX}:{sfn_name}', 'grps_ingestion_id': grps_ingestion_id})