"""Catalog ingestion helper.""" from typing import List, NamedTuple, Union from ddex_ingester_common.constants.rds_queries import ( INSERT_INTO_CATALOG_INGESTION, INSERT_INTO_CATALOG_INGESTION_ACTION, INSERT_INTO_CATALOG_INGESTION_VALIDATION_RESULT ) from lambdacommon.util import mysql_connection """ Note: If defaults other than `None` are used for fields in NamedTuple classes (such as a datetime for timestamp) then that default value may get cached between lambda invocations resulting in undefined behaviour. For information around how lambda runtimes work: https://docs.aws.amazon.com/lambda/latest/dg/runtimes-context.html """ class CatalogIngestion(NamedTuple): """Represents a catalog ingestion.""" table_name = 'catalog_ingestion' column_order = [ 'state_machine_name', 'state_machine_execution_name', 'catalog_ingestion_source_id', 'timestamp', 's3_bucket_name', 's3_key_name', 'ingest_format', 'vendor_id', 'subaccount_id', 'status', 'error_message', ] state_machine_name: str state_machine_execution_name: str s3_bucket_name: str s3_key_name: str ingest_format: str timestamp: str catalog_ingestion_source_id: int = None vendor_id: int = None subaccount_id: int = None status: str = None error_message: str = None class CatalogIngestionValidationResult(NamedTuple): """Represents a catalog ingestion validation result.""" table_name = 'catalog_ingestion_validation_result' column_order = [ 'state_machine_name', 'state_machine_execution_name', 'validation_rule_id', 'response', 'message', 'isrc', 'category' ] state_machine_name: str state_machine_execution_name: str validation_rule_id: int = None response: str = None message: str = None isrc: str = None category: str = None class CatalogIngestionAction(NamedTuple): """Represents a catalog ingestion action.""" table_name = 'catalog_ingestion_action' column_order = [ 'state_machine_name', 'state_machine_execution_name', 'time_created', 'action', 'entity_type', 'project_code', 'project_id', 'project_name', 'upc', 'release_id', 'release_name', 'vendor_catalog_number', 'isrc', 'tuid', 'track_sequence_number', 'track_volume_number', 'track_name', 'result', 'message' ] state_machine_name: str state_machine_execution_name: str action: str entity_type: str result: str time_created: str project_code: str = None project_id: int = None project_name: str = None upc: str = None release_id: int = None release_name: str = None vendor_catalog_number: str = None isrc: str = None tuid: int = None track_sequence_number: int = None track_volume_number: int = None track_name: str = None message: str = None class CatalogIngestionSession: """Class representing a catalog ingestion session. Example: Adding a single model to this session:: session = CatalogIngestionSession( 'host', 'db_name', 'user', 'password') validation_result = CatalogIngestionValidationResult( state_machine_name='foo', state_machine_execution_name='bar') session.add(validation_result) Adding multiple models to this session:: session = CatalogIngestionSession( 'host', 'db_name', 'user', 'password') results = [ CatalogIngestionValidationResult( state_machine_name='foo', state_machine_execution_name='bar' ), CatalogIngestionValidationResult( state_machine_name='foo', state_machine_execution_name='bar' ) ] session.add(results) """ def __init__( self, rds_host: str, rds_db_name: str, rds_user: str, rds_password: str): """Construct a catalog ingestion session.""" self.connection_info = { 'host': rds_host, 'database': rds_db_name, 'user': rds_user, 'password': rds_password, } self.data = { 'catalog_ingestion': [], 'catalog_ingestion_action': [], 'catalog_ingestion_validation_result': [] } self.query_map = { 'catalog_ingestion': INSERT_INTO_CATALOG_INGESTION, 'catalog_ingestion_action': INSERT_INTO_CATALOG_INGESTION_ACTION, 'catalog_ingestion_validation_result': INSERT_INTO_CATALOG_INGESTION_VALIDATION_RESULT } def add(self, models: Union[List[NamedTuple], NamedTuple]): """Add a constructed catalog ingestion model to the session.""" if not isinstance(models, list): models = [models] for model in models: self.data.get(model.table_name).append(model) def save(self): """Save the data in this session as rows in RDS.""" for table, models in self.data.items(): if models: data = [tuple(model) for model in models] # Insert data into the table with mysql_connection(**self.connection_info) as rds_conn: with rds_conn.cursor() as cursor: cursor.executemany(self.query_map[table], data) # Clear catalog ingestion data after saving # If the Lambda using this helper is cached the config is reused # This means the CatalogIngestionSession object collects all of # the data added by each Lambda invocation if we don't clear it self.data = { 'catalog_ingestion': [], 'catalog_ingestion_action': [], 'catalog_ingestion_validation_result': [] }