"""Delete assets from AWS storage and mark them deleted in database.""" import datetime import json from collections import defaultdict from typing import Any import sentry_sdk from sqlalchemy.exc import SQLAlchemyError from assets.connectors import mysql from assets.constants import ( api, asset_status as asset_status_constants, error, product as product_constants, success, ) from assets.exceptions import ( AssetDeleteFailure, AssetFinalNotFound, AssetStatusNotFound, AssetUploadNotFound, ErrorDiscardingCorrections, InvalidProductStatus, WrongAssetStatus, ) from assets.models import ( asset_final, asset_status, asset_upload as asset_upload_model, asset_upload_type, ows_product, ows_track, s3_file, ) def _group_files_by_assets( delete_message: dict[str, Any], asset_upload_ids: list[int], bucket_file_to_asset_id_mapping: dict[tuple[str, str], int], ) -> dict[int, dict[str, Any]]: """Group deleted files by asset_upload_ids. Args: delete_message (dict): Delete response body. asset_upload_ids (list): List of asset_upload ids. bucket_file_to_asset_id_mapping (dict): Mapping to get asset_upload id by (bucket, filename) pair. Returns: dict: File delete results grouped by asset_upload ids. """ asset_uploads_delete_file_mapping: dict[int, dict[str, Any]] = { asset_upload_id: {} for asset_upload_id in asset_upload_ids } for bucket, details in delete_message["details"].items(): failed = details["failed"] succeeded = details["succeeded"] error_code = details.get("error_code") succeeded_map = defaultdict(list) failed_map = defaultdict(list) for filename in failed: asset_id = bucket_file_to_asset_id_mapping[(bucket, filename["Key"])] failed_map[asset_id].append(filename) for filename in succeeded: asset_id = bucket_file_to_asset_id_mapping[(bucket, filename["Key"])] succeeded_map[asset_id].append(filename) for asset_id in succeeded_map: if bucket not in asset_uploads_delete_file_mapping[asset_id]: asset_uploads_delete_file_mapping[asset_id][bucket] = { "succeeded": [], "failed": [], } asset_uploads_delete_file_mapping[asset_id][bucket]["succeeded"] = ( succeeded_map[asset_id] ) for asset_id in failed_map: if bucket not in asset_uploads_delete_file_mapping[asset_id]: asset_uploads_delete_file_mapping[asset_id][bucket] = { "succeeded": [], "failed": [], } asset_uploads_delete_file_mapping[asset_id][bucket]["failed"] = failed_map[ asset_id ] if error_code: asset_uploads_delete_file_mapping[asset_id][bucket]["error_code"] = ( error_code ) return asset_uploads_delete_file_mapping def _is_done_processing(status: dict[str, Any]) -> bool: """Check if asset status reflects processing completed state. Args: status (dict): Asset upload item status Returns: bool: Successful response indicates asset processing is done. """ if status["status"] in asset_status_constants.PROCESSING_FINISHED_ASSET_STATUSES: return True if status["status"] is None: return True return False def _check_assets_ready_to_delete(assets_data: list[dict[str, Any]]) -> None: """Check if assets can be deleted. Args: assets_data (list): each item is an asset_upload dict. """ asset_upload_by_id = {} product_tuid_to_asset_upload_id_map = defaultdict(list) for asset_upload_item in assets_data: asset_upload_id = asset_upload_item["id"] key = (asset_upload_item["product_id"], asset_upload_item["track_unique_id"]) product_tuid_to_asset_upload_id_map[key].append(asset_upload_id) asset_upload_by_id[asset_upload_id] = asset_upload_item asset_upload_ids_to_check_statuses = [] for key, asset_upload_ids in product_tuid_to_asset_upload_id_map.items(): latest_id = max(asset_upload_ids) asset_upload_ids_to_check_statuses.append(latest_id) asset_statuses = asset_status.get_asset_statuses_by_asset_upload_ids( sorted(asset_upload_ids_to_check_statuses) ) if not asset_statuses: raise AssetStatusNotFound(error.ERROR_ASSET_STATUS_NOT_FOUND) wrong_file_statuses = [] for asset_upload_id in asset_upload_ids_to_check_statuses: status = asset_statuses.get(asset_upload_id) status = status or {"status": None} if not _is_done_processing(status): wrong_file_statuses.append( { "asset": asset_upload_by_id[asset_upload_id], "status": status["status"], } ) if wrong_file_statuses: raise WrongAssetStatus(error.ERROR_CODE_ASSET_NOT_READY_FOR_DELETE) def _get_s3_files_to_delete( asset_final_items: dict[int, list[dict[str, Any]]], asset_uploads_data: list[dict[str, Any]], ) -> tuple[list[dict[str, Any]], dict[tuple[str, str], int]]: """Return a list of bucket and filename dicts. Args: asset_final_items (dict): Dict of asset_upload_id->asset_upload items. asset_uploads_data (list): List of asset_upload items. Returns: tuple: list of files to delete and file mapping between bucket and asset_upload_id """ files_to_delete = [] bucket_file_to_asset_upload_id_mapping: dict[tuple[str, str], int] = {} for asset_upload_id, asset_final_data in asset_final_items.items(): for asset_final_item in asset_final_data: bucket = asset_final_item["bucket"] filename = asset_final_item["filename"] files_to_delete.append({"bucket": bucket, "file": filename}) bucket_file_to_asset_upload_id_mapping[(bucket, filename)] = int( asset_upload_id ) return files_to_delete, bucket_file_to_asset_upload_id_mapping def delete_assets(asset_uploads_data: list[dict[str, Any]]) -> dict[str, Any]: """Delete assets from S3 and marks db records as deleted. Args: asset_uploads_data (list): List of asset_upload items. Returns: dict: Response containing file removing status. """ _check_assets_ready_to_delete(asset_uploads_data) asset_upload_ids = [asset_upload["id"] for asset_upload in asset_uploads_data] mark_deleted_response = asset_upload_model.mark_asset_upload_deleted( asset_upload_ids ) try: asset_final_response = asset_final.get_asset_final_by_asset_upload_ids( asset_upload_ids ) except AssetFinalNotFound: asset_final_response = {} files_to_delete, bucket_file_to_asset_upload_id_mapping = _get_s3_files_to_delete( asset_final_response, asset_uploads_data ) delete_status = s3_file.delete_s3_files(files_to_delete) if any(delete_status[bucket_key]["failed"] for bucket_key in delete_status.keys()): delete_message = {"status": "failure", "details": delete_status} else: delete_message = {"status": "ok", "details": delete_status} asset_uploads_delete_file_mapping = _group_files_by_assets( delete_message, asset_upload_ids, bucket_file_to_asset_upload_id_mapping ) failed_asset_uploads = [] succeeded_asset_uploads = [] asset_upload_id_to_asset_upload_marked_mapping = { asset_upload["id"]: asset_upload for asset_upload in mark_deleted_response } for ( asset_upload_id, delete_details, ) in asset_uploads_delete_file_mapping.items(): asset_upload = asset_upload_id_to_asset_upload_marked_mapping[asset_upload_id] asset_delete_info = { "asset_upload": asset_upload, "delete_details": delete_details, } delete_error = any( delete_details[bucket]["failed"] for bucket in delete_details.keys() ) if delete_error: status = asset_status_constants.STATUS_DELETE_FAILED failed_asset_uploads.append(asset_delete_info) else: status = asset_status_constants.STATUS_DELETE_SUCCEEDED succeeded_asset_uploads.append(asset_delete_info) try: asset_status.create_asset_status( int(asset_upload_id), status, datetime.datetime.now(), message=json.dumps(delete_details), ) except SQLAlchemyError as e: sentry_sdk.capture_exception(e) message = {"succeeded": succeeded_asset_uploads, "failed": failed_asset_uploads} if failed_asset_uploads: raise AssetDeleteFailure(message) return message def delete_track( track_id: int, asset_upload_type_name: str | None = None ) -> dict[str, Any]: """Delete track. Args: track_id (int): Track unique id. asset_upload_type_name (str | None): Optional asset upload type to filter by. Returns: dict: delete assets response. """ track = ows_track.get_track_by_id(track_id) product = ows_product.get_product_by_id(track["product_id"]) if product["status"] == product_constants.PRODUCT_STATUS_IN_CONTENT: raise InvalidProductStatus(error.ERROR_MESSAGE_PRODUCT_IN_CONTENT) asset_upload_type_id = ( asset_upload_type.resolve_asset_upload_type_id(asset_upload_type_name) if asset_upload_type_name is not None else None ) asset_upload_response = asset_upload_model.get_asset_uploads( track_id=track_id, api_version=api.API_VERSION_V2, asset_upload_type_id=asset_upload_type_id, ) return delete_assets(asset_upload_response) def delete_product(product_id: int) -> dict[str, Any]: """Delete product. Args: product_id (int): Product id. Returns: dict: delete assets response. """ product = ows_product.get_product_by_id(product_id) if product["status"] == product_constants.PRODUCT_STATUS_IN_CONTENT: raise InvalidProductStatus(error.ERROR_MESSAGE_PRODUCT_IN_CONTENT) asset_uploads_response = asset_upload_model.get_asset_uploads( product_id=product_id, api_version=api.API_VERSION_V2 ) return delete_assets(asset_uploads_response) def delete_product_corrections(product_id: int) -> str: """Delete product corrections. Args: product_id (int): Product id. Returns: str: success message. """ with mysql.au_db_session() as session: try: asset_uploads = asset_upload_model.get_asset_uploads_records( session=session, api_version=api.API_VERSION_V2, product_id=product_id, is_correction=1, ) asset_upload_ids = [asset_upload["id"] for asset_upload in asset_uploads] updated_asset_uploads = ( asset_upload_model.mark_asset_upload_records_deleted( session=session, asset_upload_ids=asset_upload_ids ) ) if updated_asset_uploads != len(asset_upload_ids): raise ErrorDiscardingCorrections( error.ERROR_DISCARDING_CORRECTIONS.format( product_id=product_id, updated=updated_asset_uploads, expected=len(asset_upload_ids), ) ) session.commit() except AssetUploadNotFound as e: raise AssetUploadNotFound( error.ERROR_NO_CORRECTIONS_TO_DISCARD.format(product_id) ) from e return success.DISCARDING_CORRECTION_SUCCESS.format(product_id)