"""awal-ingestion-flow-control.""" from time import sleep from ddex_ingester_common.constants.ddex_providers import ( ALTAFONTE, ALTAFONTE_FOLDER_NAME, RISING_88, RISING_88_FOLDER_NAME) from ddex_ingester_common.helpers.rds import run_rds_query import config import sfn from config import secrets_manager_client from constants import constants, queries logger = config.get_current_logger(config.app_logger) def handler(event, context): """Begin main script execution function.""" logger.info( f'Beginning ingestion flow control. Event: {event} Context: {context}') # The maximum number of state machines to have running at the same time max_sfn_count = event.get('MAX_STATE_MACHINES') or constants.MAX_STATE_MACHINES # noqa # The delay between each Lambda batch invocation sfn_start_delay = event.get('SFN_START_DELAY') or constants.SFN_START_DELAY # The number of executions to start for each sfn_start_delay sfn_batch_size = event.get('SFN_BATCH_SIZE') or constants.SFN_BATCH_SIZE logger.info( f'MAX_STATE_MACHINES={max_sfn_count} ' f'SFN_START_DELAY={sfn_start_delay} ' f'SFN_BATCH_SIZE={sfn_batch_size}' ) # We need to make sure that the (sfn_start_delay * max_sfn_count) time # is lower than the time between each flow control run to avoid having # more than one flow control run at a time # Get the count of current executions current_sfn_count = sfn.get_full_execution_count_by_arn( config.AWAL_INGESTION_SFN_ARN, constants.EXECUTION_RUNNING_STATUS ) # Check if you can run more State machines (too many concurrent executions) nr_ingests = max_sfn_count - current_sfn_count if nr_ingests < 1: logger.info( f'Cannot run any more state machines. Currently running ' f'{current_sfn_count} with a limit of {max_sfn_count}' ) return logger.info( f'Currently running {current_sfn_count} executions with a limit of ' f'{max_sfn_count}. Starting at most {nr_ingests} executions' ) # Get the name of current executions current_exec_name_list = sfn.get_full_execution_name_list_by_arn( config.AWAL_INGESTION_SFN_ARN, constants.EXECUTION_RUNNING_STATUS ) rows_to_ingest = run_rds_query( logger, config.RDS_HOST, config.RDS_DB_NAME, config.RDS_RW_USER, secrets_manager_client.get_cred('rds_read_write_password'), queries.SELECT_ALL_N_ROWS_ORDERED, nr_ingests ) if not rows_to_ingest: logger.info('No deliveries to ingest. Ending ingestion flow control.') return for i, row in enumerate(rows_to_ingest): bucket = row.get('s3_bucket_name') key = row.get('s3_key_name') logger.info( f'Starting execution of ARN {config.AWAL_INGESTION_SFN_ARN}, ' f'with Bucket {bucket} and Key {key}' ) ddex_provider = config.AWAL_DDEX_PROVIDER if RISING_88_FOLDER_NAME in key: ddex_provider = RISING_88 elif ALTAFONTE_FOLDER_NAME in key: ddex_provider = ALTAFONTE sfn.start_execution_by_arn( config.AWAL_INGESTION_SFN_ARN, bucket, key, ddex_provider, row.get('ddex_delivery_to_ingest_id') ) logger.info(f'Updating row status for key {key}') run_rds_query( logger, config.RDS_HOST, config.RDS_DB_NAME, config.RDS_RW_USER, secrets_manager_client.get_cred('rds_read_write_password'), queries.UPDATE_ROW_STATUS, row.get('ddex_delivery_to_ingest_id'), ) if (i + 1) % sfn_batch_size == 0: # Sleeping to let sfn batch start. logger.info(f'Sleeping for {sfn_start_delay} milliseconds.') sleep(sfn_start_delay / 1000) # Get the count of current executions again after starting the new ones full_exec_name_list = sfn.get_full_execution_name_list_by_arn( config.AWAL_INGESTION_SFN_ARN, constants.EXECUTION_RUNNING_STATUS ) # Get the names of all the newly started executions new_exec_name_list = [ x for x in full_exec_name_list if x not in current_exec_name_list] for name in new_exec_name_list: logger.info(f'Execution {name} was started.') logger.info('End of ingestion flow control.')