"""S3 upload service - handles file uploads to S3 buckets.""" import logging from typing import Optional, Callable import boto3 from botocore.exceptions import ClientError, NoCredentialsError from backfillapi import config logger = logging.getLogger(__name__) class S3UploadError(Exception): """Custom exception for S3 upload errors.""" def __init__(self, message: str, code: Optional[str] = None, status_code: Optional[int] = None): super().__init__(message) self.code = code self.status_code = status_code def get_s3_client(): """Create boto3 S3 client with credentials from environment.""" return boto3.client('s3', region_name=config.AWS_REGION) def construct_s3_key(folder_path: str, filename: str) -> str: """ Construct S3 key from folder path and filename. Args: folder_path: Folder path (e.g., "PP-1055") filename: File name Returns: Full S3 key (e.g., "PP-1055/permissions.csv") """ clean_path = folder_path.strip().strip('/') return f"{clean_path}/{filename}" if clean_path else filename def validate_folder_path(folder_path: str) -> Optional[str]: """ Validate folder path for S3 upload. Returns: Error message if invalid, None if valid """ trimmed = folder_path.strip() if not trimmed: return "Folder path cannot be empty" if trimmed.startswith('/') or trimmed.endswith('/'): return "Folder path should not start or end with a slash" # Reserved paths reserved_paths = ['pdp_qa_refresh', 'integration-test'] if trimmed in reserved_paths: return f"'{trimmed}' is a reserved folder path and cannot be used" # Invalid S3 key characters invalid_chars = ['<', '>', '{', '}', '[', ']', '\\', '^', '%', '`', '|', '"'] if any(char in trimmed for char in invalid_chars): return "Folder path contains invalid characters" return None def upload_file_to_s3( s3_client, file_content: bytes, filename: str, s3_key: str, bucket: str, content_type: str = 'application/octet-stream' ) -> str: """ Upload a single file to S3. Args: s3_client: Boto3 S3 client file_content: File content as bytes filename: Original filename (for error messages) s3_key: Full S3 key (path) bucket: S3 bucket name content_type: MIME type Returns: The S3 key of uploaded file Raises: S3UploadError: If upload fails """ try: s3_client.put_object( Bucket=bucket, Key=s3_key, Body=file_content, ContentType=content_type ) logger.info(f"Successfully uploaded {filename} to s3://{bucket}/{s3_key}") return s3_key except NoCredentialsError: raise S3UploadError( "AWS credentials not found. Please configure AWS CLI or check environment variables.", code="NoCredentials", status_code=401 ) except ClientError as e: error_code = e.response.get('Error', {}).get('Code', 'Unknown') if error_code == 'NoSuchBucket': raise S3UploadError( f"Bucket '{bucket}' does not exist", code="NoSuchBucket", status_code=404 ) elif error_code == 'AccessDenied': raise S3UploadError( f"Access denied when uploading '{filename}'. Check AWS permissions.", code="AccessDenied", status_code=403 ) elif error_code == 'ExpiredToken': raise S3UploadError( f"AWS credentials have expired. If using Docker, restart the containers with 'docker-compose down && awsume permissions-platform-{{qa|prod}}-generic && docker-compose up'. If running locally, re-run 'awsume permissions-platform-{{qa|prod}}-generic' and restart the backend.", code="ExpiredToken", status_code=401 ) else: raise S3UploadError( f"Failed to upload '{filename}': {str(e)}", code=error_code ) def upload_files( csv_files: list[tuple[str, bytes, str]], # (filename, content, content_type) manifest_file: Optional[tuple[str, bytes, str]], folder_path: str, bucket: str, progress_callback: Optional[Callable[[str, int, int], None]] = None ) -> dict: """ Upload CSV files and manifest to S3. Strategy: Upload CSV files first, then manifest last (manifest triggers backfill). Args: csv_files: List of (filename, content, content_type) tuples manifest_file: Optional (filename, content, content_type) tuple folder_path: S3 folder path bucket: S3 bucket name progress_callback: Optional callback(filename, current, total) Returns: { "status": "success" | "error", "message": str, "uploaded_files": List[str], "errors": Optional[List[str]] } """ s3_client = get_s3_client() uploaded_files = [] errors = [] try: # Upload CSV files first total_csv = len(csv_files) for i, (filename, content, content_type) in enumerate(csv_files, start=1): s3_key = construct_s3_key(folder_path, filename) if progress_callback: progress_callback(filename, i, total_csv) upload_file_to_s3(s3_client, content, filename, s3_key, bucket, content_type) uploaded_files.append(s3_key) # Upload manifest last if manifest_file: filename, content, content_type = manifest_file s3_key = construct_s3_key(folder_path, filename) if progress_callback: progress_callback(filename, 1, 1) upload_file_to_s3(s3_client, content, filename, s3_key, bucket, content_type) uploaded_files.append(s3_key) return { "status": "success", "message": f"Successfully uploaded {len(uploaded_files)} file(s) to S3", "uploaded_files": uploaded_files } except S3UploadError as e: errors.append(str(e)) return { "status": "error", "message": str(e), "uploaded_files": uploaded_files if uploaded_files else None, "errors": errors, "error_code": e.code, "status_code": e.status_code or 500 } except Exception as e: logger.exception("Unexpected error during upload") errors.append(str(e)) return { "status": "error", "message": f"Upload failed: {str(e)}", "uploaded_files": uploaded_files if uploaded_files else None, "errors": errors, "error_code": "UnknownError", "status_code": 500 }