"""Lambda generate-error-report function module.""" import json import uuid from typing import Any import boto3 import pandas as pd from mypy_boto3_s3.client import S3Client from mypy_boto3_s3.paginator import ListObjectsV2Paginator from owslogger.logger import OwsLoggingAdapter from src.config import CSV_DELIMITER, S3_INPUT_BUCKET, S3_OUTPUT_BUCKET, app_logger s3_client: S3Client = boto3.client("s3") def handler(event: Any, context: Any) -> dict[str, str]: """Lambda Entrypoint.""" logger = OwsLoggingAdapter(app_logger, {"correlation_id": uuid.uuid4()}) export_id = event["export_id"] execution_name = event["execution_name"] prefix = f"{export_id}/{execution_name}" paginator: ListObjectsV2Paginator = s3_client.get_paginator("list_objects_v2") page_iterator = paginator.paginate(Bucket=S3_INPUT_BUCKET, Prefix=prefix) log_msg = f"Generating error reports in CSV for export: {export_id} - execution name: {execution_name}" logger.info(log_msg) errors: list[dict[str, str]] = [] for page in page_iterator: for contents in page["Contents"]: file_key = contents.get("Key") if file_key is not None: if file_key.find("asset_copy_errors") != -1 and file_key.endswith( ".json" ): data = get_errors_from_s3(file_key) errors.extend(data) if errors: write_csv_errors_to_s3(errors, export_id, logger) return {"export_id": export_id, "execution_name": execution_name} def get_errors_from_s3(file_key: str) -> list[dict[str, str]]: file_obj = s3_client.get_object(Bucket=S3_INPUT_BUCKET, Key=file_key) file_content = file_obj["Body"].read().decode("utf-8") data = json.loads(file_content) # Remove source location (internal asset location) - not required in a user-facing error report filtered_data = [ {key: val for key, val in d.items() if key != "source"} for d in data ] return filtered_data def write_csv_errors_to_s3( errors: list[dict[str, str]], export_id: str, logger: OwsLoggingAdapter ) -> None: json_str = json.dumps(errors) json_data = pd.read_json(json_str) csv_data = json_data.to_csv( encoding="utf-8", sep=CSV_DELIMITER, index=False, header=["asset", "error_reason"], ) filename = f"{export_id}/error_report.csv" try: s3_client.put_object( Bucket=S3_OUTPUT_BUCKET, Key=filename, Body=csv_data, ContentType="text/csv", ) except Exception as e: error_class_name = e.__class__.__name__ error_msg = f"Failed to upload CSV error report for export: {export_id} {error_class_name}: {str(e)}" logger.error(error_msg)