"""Logic for interacting with RDS.""" import logging import time from botocore.exceptions import ClientError from lambdacommon.common_config import logger from tenacity import before_sleep_log from tenacity import retry_if_exception_type from tenacity import Retrying from tenacity import stop_after_attempt from tenacity import wait_fixed import config from src.connectors.boto_clients import rds_client as client from src.utils import password def find_database_identifier(db_name): """Determine database identifier from supplied name.""" response = client.describe_db_instances( Filters=[{'Name': 'db-cluster-id', 'Values': [db_name]}] ) if response['DBInstances']: logger.info(f'Database {db_name} is an instance in a cluster') db_type = 'cluster' return db_name, db_type else: # See if it is a non-cluster standalone instance response = client.describe_db_instances( Filters=[{'Name': 'db-instance-id', 'Values': [db_name]}]) if response['DBInstances']: logger.info(f'Database {db_name} is a standalone instance') db_type = 'standalone' return response['DBInstances'][0]['DBInstanceIdentifier'], db_type else: raise DatabaseNotFound('Database not found') def reset_rds_master_credentials(db_name, db_type, wait=True): """ Lookup RDS cluster or instance and reset master credentials for it. Args: db_name (str): the name of the db, in the form of env-service-name, e.g. qa-ows-assets db_type (str): the database type (cluster, standalone) Returns: dict: newly reset user credentials """ if db_type == 'cluster': response = client.describe_db_clusters( Filters=[{'Name': 'db-cluster-id', 'Values': [db_name]}] ) connection_credentials = { 'host': response['DBClusters'][0]['Endpoint'], 'username': response['DBClusters'][0]['MasterUsername'], 'password': password.generate_random_password(), } # Wait for cluster to be available to ensure modify operation doesn't # fail (e.g. due to automated backups taking place) client.get_waiter(waiter_name='db_cluster_available').wait( DBClusterIdentifier=db_name) # Retry on InvalidDBClusterStateFault: availability check can succeed # while a concurrent operation (e.g. snapshot) has already been issued, # causing modify_db_cluster to fail due to a cluster state race. try: for attempt in Retrying( stop=stop_after_attempt(config.PASSWORD_RESET_RETRY_LIMIT), wait=wait_fixed(config.PASSWORD_RESET_RETRY_DELAY), retry=retry_if_exception_type( client.exceptions.InvalidDBClusterStateFault ), before_sleep=before_sleep_log(logger, logging.INFO), reraise=True, ): with attempt: client.modify_db_cluster( DBClusterIdentifier=db_name, ApplyImmediately=True, MasterUserPassword=connection_credentials['password'], ) except ClientError as error: raise SystemExit(f'Error resetting master password: {error}') elif db_type == 'standalone': response = client.describe_db_instances( DBInstanceIdentifier=db_name) connection_credentials = { 'host': response['DBInstances'][0]['Endpoint']['Address'], 'username': response['DBInstances'][0]['MasterUsername'], 'password': password.generate_random_password(), } # Wait for instance to be available to ensure modify operation doesn't # fail (e.g. due to automated backups taking place) client.get_waiter(waiter_name='db_instance_available').wait( DBInstanceIdentifier=db_name) # Retry on InvalidDBInstanceStateFault for the same race condition # that can occur with clusters (see above). try: for attempt in Retrying( stop=stop_after_attempt(config.PASSWORD_RESET_RETRY_LIMIT), wait=wait_fixed(config.PASSWORD_RESET_RETRY_DELAY), retry=retry_if_exception_type( client.exceptions.InvalidDBInstanceStateFault ), before_sleep=before_sleep_log(logger, logging.INFO), reraise=True, ): with attempt: client.modify_db_instance( DBInstanceIdentifier=db_name, ApplyImmediately=True, MasterUserPassword=connection_credentials['password'], ) except ClientError as error: raise SystemExit(f'Error resetting master password: {error}') else: raise UnsupportedDatabaseType(f'db_type {db_type} not found') if wait: # Wait until password reset has finished wait_for_pending_modifications(db_name, db_type) return connection_credentials def _poll_for_pending_modifications(db_name, db_type): """ Poll for pending changes to cluster. Returns: bool: existence of pending modifications """ if db_type == 'cluster': response = client.describe_db_clusters( Filters=[{'Name': 'db-cluster-id', 'Values': [db_name]}] ) cluster = response['DBClusters'][0] if 'PendingModifiedValues' in cluster \ and cluster['PendingModifiedValues']: logger.info(f'There are pending changes for {db_name}.') return True else: logger.info(f'No pending changes for {db_name}.') return False elif db_type == 'standalone': response = client.describe_db_instances( DBInstanceIdentifier=db_name, ) instance = response['DBInstances'][0] if 'PendingModifiedValues' in instance \ and instance['PendingModifiedValues']: logger.info(f'There are pending changes for {db_name}.') return True else: logger.info(f'No pending changes for {db_name}.') return False else: raise UnsupportedDatabaseType(f'db_type {db_type} not found') def wait_for_pending_modifications(db_name, db_type): """Wait for pending modifications to the given database to complete.""" timeout = time.time() + config.PENDING_CHANGES_WAIT_TIMEOUT while time.time() < timeout: if _poll_for_pending_modifications(db_name, db_type): logger.info( 'Database modifications are pending. Waiting 10 seconds...') time.sleep(10) else: logger.info('Database modifications have completed') return raise Timeout('Timeout reached and there are still pending changes.') class Timeout(Exception): """Exception when an operation times out.""" pass class UnsupportedDatabaseType(Exception): """Exception when a database type is not supported.""" pass class DatabaseNotFound(Exception): """Exception when a database cannot be found.""" pass