"""Lambda songwhip_profile_creator function module.""" import csv import io import json from dataclasses import dataclass from datetime import datetime, timezone from typing import Any import boto3 import config from config import connector_neo4j from src.constants import ( EXPECTED_BUCKET_OWNER, IDENTITY_EXISTS_QUERY, REQUIRED_COLUMNS, TENANT_TYPE_TO_LABEL, get_create_profile_query, get_tenant_exists_query, ) @dataclass class InputRow: identity_id: str tenant_type: str tenant_uuid: str @dataclass class BadRow: row: InputRow reason: str def parse_s3_event(event: dict) -> tuple[str, str]: """Extract bucket and key from S3 event.""" try: detail = event["detail"] bucket_name = detail["bucket"]["name"] object_key = detail["object"]["key"] return bucket_name, object_key except (KeyError, TypeError) as e: raise ValueError(f"Invalid EventBridge S3 event structure: {str(e)}") def read_csv_from_s3(bucket_name: str, object_key: str) -> list[InputRow]: """Download and parse CSV file from S3.""" s3_client = boto3.client("s3") config.logger.info(f"Reading CSV from s3://{bucket_name}/{object_key}") response = s3_client.get_object( Bucket=bucket_name, Key=object_key, ExpectedBucketOwner=EXPECTED_BUCKET_OWNER ) csv_content = response["Body"].read().decode("utf-8") csv_reader = csv.DictReader(io.StringIO(csv_content)) # Validate headers if csv_reader.fieldnames is None: raise ValueError("CSV file has no headers") missing_columns = set(REQUIRED_COLUMNS) - set(csv_reader.fieldnames) if missing_columns: raise ValueError(f"CSV missing required columns: {missing_columns}") rows = [] for line_num, row in enumerate(csv_reader, start=2): rows.append( InputRow( identity_id=row["identity_id"].strip(), tenant_type=row["tenant_type"].strip().lower(), tenant_uuid=row["tenant_uuid"].strip(), ) ) config.logger.info(f"Successfully read {len(rows)} rows from CSV") return rows def validate_row(row: InputRow) -> str | None: """Validate a single input row. Returns error message or None if valid.""" if not row.identity_id: return "identity_id is empty" if not row.tenant_type: return "tenant_type is empty" if not row.tenant_uuid: return "tenant_uuid is empty" if row.tenant_type not in TENANT_TYPE_TO_LABEL: return f"invalid tenant_type: {row.tenant_type}" return None def check_identity_exists(identity_id: str, session: Any) -> bool: """Check if identity exists in Neo4j.""" result = session.run(IDENTITY_EXISTS_QUERY, identity_id=identity_id) return result.single() is not None def check_tenant_exists(tenant_uuid: str, tenant_label: str, session: Any) -> bool: """Check if tenant exists in Neo4j.""" result = session.run(get_tenant_exists_query(tenant_label), tenant_uuid=tenant_uuid) return result.single() is not None def create_songwhip_profile(row: InputRow, session: Any) -> bool: """ Create a SongwhipProfile for the given identity and tenant. Returns True if profile was created/exists, False otherwise. """ tenant_label = TENANT_TYPE_TO_LABEL[row.tenant_type] result = session.run( get_create_profile_query(tenant_label), identity_id=row.identity_id, tenant_uuid=row.tenant_uuid, ) return result.single() is not None def process_row(row: InputRow, session: Any) -> BadRow | None: """ Process a single row: validate, check existence, create profile. Returns BadRow if there was an error, None if successful. """ validation_error = validate_row(row) if validation_error: config.logger.warning( f"Validation failed for row {row.identity_id}: {validation_error}" ) return BadRow(row=row, reason=validation_error) tenant_label = TENANT_TYPE_TO_LABEL[row.tenant_type] # Check if identity exists if not check_identity_exists(row.identity_id, session): config.logger.warning(f"Identity not found: {row.identity_id}") return BadRow(row=row, reason="identity_not_found") config.logger.info(f"Identity found: {row.identity_id}") # Check if tenant exists if not check_tenant_exists(row.tenant_uuid, tenant_label, session): config.logger.warning( f"Tenant not found: {row.tenant_uuid} (type: {row.tenant_type})" ) return BadRow(row=row, reason="tenant_not_found") config.logger.info(f"Tenant found: {row.tenant_uuid}") # Create the profile if not create_songwhip_profile(row, session): config.logger.error(f"Failed to create profile for identity {row.identity_id}") return BadRow(row=row, reason="profile_creation_failed") config.logger.info(f"Profile created for identity {row.identity_id}") return None def generate_pdp_csv_to_s3( successful_rows: list[InputRow], bucket_name: str, timestamp: str ) -> tuple[str, str]: """Generate a CSV file for PDP backfill and upload it to S3.""" csv_buffer = io.StringIO() fieldnames = ["identity_uuid", "tenant_uuid", "tenant_type", "role", "operation"] writer = csv.DictWriter(csv_buffer, fieldnames=fieldnames) writer.writeheader() for row in successful_rows: writer.writerow( { "identity_uuid": row.identity_id, "tenant_uuid": row.tenant_uuid, "tenant_type": row.tenant_type, "role": "songwhip_read", "operation": "attach", } ) s3_client = boto3.client("s3") csv_content = csv_buffer.getvalue() key = f"{config.SONGWHIP_OUTPUT_FOLDER}{timestamp}/{timestamp}.csv" s3_client.put_object( Bucket=bucket_name, Key=key, Body=csv_content, ContentType="text/csv", ExpectedBucketOwner=EXPECTED_BUCKET_OWNER, ) config.logger.info(f"PDP CSV uploaded to s3://{bucket_name}/{key}") return f"s3://{bucket_name}/{key}", timestamp def generate_bad_rows_csv_to_s3( bad_rows: list[BadRow], bucket_name: str, timestamp: str ) -> str: """Generate a CSV file for failed rows and upload it to S3.""" csv_buffer = io.StringIO() fieldnames = ["identity_id", "tenant_type", "tenant_uuid", "error_reason"] writer = csv.DictWriter(csv_buffer, fieldnames=fieldnames) writer.writeheader() for bad_row in bad_rows: writer.writerow( { "identity_id": bad_row.row.identity_id, "tenant_type": bad_row.row.tenant_type, "tenant_uuid": bad_row.row.tenant_uuid, "error_reason": bad_row.reason, } ) s3_client = boto3.client("s3") csv_content = csv_buffer.getvalue() key = f"{config.SONGWHIP_OUTPUT_FOLDER}{timestamp}/failed_rows.csv" s3_client.put_object( Bucket=bucket_name, Key=key, Body=csv_content, ContentType="text/csv", ExpectedBucketOwner=EXPECTED_BUCKET_OWNER, ) config.logger.info(f"Failed rows CSV uploaded to s3://{bucket_name}/{key}") return f"s3://{bucket_name}/{key}" def generate_manifest_json_to_s3(timestamp: str, bucket_name: str) -> str: """Generate and upload manifest.json file to S3.""" manifest = { "bucket": f"{config.ENVIRONMENT}-pdp-backfill", "jobs": [ { "job_type": "attach_and_detach", "keys": [f"songwhip-backfill/{timestamp}.csv"], } ], } s3_client = boto3.client("s3") manifest_content = json.dumps(manifest, indent=2) manifest_key = f"{config.SONGWHIP_OUTPUT_FOLDER}{timestamp}/manifest.json" s3_client.put_object( Bucket=bucket_name, Key=manifest_key, Body=manifest_content, ContentType="application/json", ExpectedBucketOwner=EXPECTED_BUCKET_OWNER, ) config.logger.info(f"Manifest file uploaded to s3://{bucket_name}/{manifest_key}") return f"s3://{bucket_name}/{manifest_key}" @connector_neo4j.Neo4jSession(transaction=True, use_v2=True, database="graph.db") def handler(event: Any, context: Any) -> dict[str, Any]: """Lambda entry point.""" try: session = connector_neo4j.get_session() bucket, key = parse_s3_event(event) rows = read_csv_from_s3(bucket, key) successful_rows: list[InputRow] = [] bad_rows: list[BadRow] = [] for row in rows: bad_row = process_row(row, session) if bad_row: bad_rows.append(bad_row) else: successful_rows.append(row) config.logger.info( f"Processed {len(rows)} rows: {len(successful_rows)} successful, " f"{len(bad_rows)} failed" ) timestamp = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H-%M-%S") result: dict[str, Any] = { "total_rows": len(rows), "successful_rows": len(successful_rows), "failed_rows": len(bad_rows), } if successful_rows: s3_url, timestamp = generate_pdp_csv_to_s3( successful_rows, config.ALTAFONTE_USER_CREATION_S3_BUCKET, timestamp ) manifest_url = generate_manifest_json_to_s3( timestamp, config.ALTAFONTE_USER_CREATION_S3_BUCKET ) result["pdp_csv_url"] = s3_url result["manifest_url"] = manifest_url if bad_rows: bad_rows_url = generate_bad_rows_csv_to_s3( bad_rows, config.ALTAFONTE_USER_CREATION_S3_BUCKET, timestamp ) result["failed_rows_url"] = bad_rows_url return result except Exception as e: config.logger.exception(str(e)) raise e