"""S3 Transfer Module for SAP Settlement Feed. This module provides functionality to upload SAP Settlement feed files to an AWS S3 bucket. It is designed to be invoked after feed file generation and gzip compression. This module mirrors the structure of sftp_transfer.py to maintain consistency in the codebase. Guiding principles (see repository AI instructions): - Minimal invasive changes: standalone module, optional invocation. - Robust logging of milestones and errors; never log secrets. - Retry with backoff for transient failures. - AWS credentials injected from Secret Manager into environment. - Mock mode support for testing without real AWS connections. - Ready for future async/concurrency although current implementation performs synchronous uploads to keep complexity low. Unit tests will mock boto3 classes; this module MUST NOT attempt any real network connections in test or CI environments. """ from __future__ import annotations from dataclasses import dataclass, field from pathlib import Path import logging import os import time from typing import Iterable, List, Optional, Sequence import boto3 # type: ignore from botocore.exceptions import ClientError, NoCredentialsError # type: ignore __all__ = [ "S3Config", "S3TransferManager", "S3UploadResult", "S3Error", "S3ConfigurationError", "S3UploadError", "transfer_to_s3_if_enabled", ] logger = logging.getLogger( os.environ.get("LOGGER_NAME", "sme-feed-file-exporter") ) class S3Error(Exception): """Base S3 exception.""" class S3ConfigurationError(S3Error): """Raised when configuration is invalid.""" class S3UploadError(S3Error): """Raised when a file upload ultimately fails after retries.""" @dataclass(frozen=True) class S3Config: """Immutable configuration for S3 transfers. Attributes: bucket: S3 bucket name. folder: S3 folder/prefix within the bucket. region: AWS region (optional, uses default if not specified). retries: Number of retry attempts after the initial try. backoff_sec: Base seconds for exponential backoff. timeout: Connection timeout in seconds. continue_on_error: If True, failures are collected and logged; otherwise first fatal failure raises and stops further uploads. """ bucket: str folder: str = "" region: str = "us-east-1" retries: int = 3 backoff_sec: float = 2.0 timeout: float = 60.0 continue_on_error: bool = False # Internal field to store sanitized folder path _sanitized_folder: str = field(init=False, repr=False) def __post_init__(self) -> None: """Validate configuration and sanitize folder path.""" # Basic validation if not self.bucket: raise S3ConfigurationError("S3 bucket is required") # Sanitize folder path: remove leading/trailing slashes folder = self.folder.strip().replace("\\", "/") if ".." in folder: raise S3ConfigurationError( "Folder path contains illegal traversal component '..'" ) # Remove leading/trailing slashes for consistent concatenation folder = folder.strip("/") object.__setattr__(self, "_sanitized_folder", folder) @property def sanitized_folder(self) -> str: """Return sanitized folder path safe for S3 key construction.""" return self._sanitized_folder @dataclass class S3UploadResult: """Summary information about an upload batch.""" uploaded: int failed: List[Path] total: int s3_keys: List[str] = field(default_factory=list) def success(self) -> bool: """Return True if all files were uploaded successfully.""" return self.uploaded == self.total and not self.failed class S3TransferManager: """Manager responsible for uploading files to S3. Usage: cfg = S3Config(bucket='my-bucket', folder='my-prefix') mgr = S3TransferManager(cfg) mgr.connect() result = mgr.upload_files(list_of_paths) mgr.close() """ def __init__(self, config: S3Config) -> None: """Initialize transfer manager with provided configuration. Args: config: S3Config instance with bucket and transfer settings. """ self.config = config self._s3_client = None self._connected = False def connect(self) -> None: """Establish S3 client connection. In mock mode (S3_MOCK=True), creates a dummy client that simulates uploads for testing. """ if self._connected: logger.debug( "S3TransferManager.connect() called but already connected." ) return # Mock mode for testing mock_flag = str(os.environ.get("S3_MOCK", "False")).lower() in { "1", "true", "yes", } if mock_flag: logger.info("S3 mock mode enabled; simulating transfers.") self._s3_client = _MockS3Client() self._connected = True return logger.info( "Establishing S3 connection to bucket '%s' in region '%s'", self.config.bucket, self.config.region, ) try: # Create S3 client using default credential chain # (env vars, IAM role, ~/.aws/credentials) self._s3_client = boto3.client( "s3", region_name=self.config.region, ) # Verify bucket access with a head_bucket call self._s3_client.head_bucket(Bucket=self.config.bucket) self._connected = True logger.info( "S3 connection established. Bucket '%s' is accessible.", self.config.bucket, ) except NoCredentialsError as exc: raise S3ConfigurationError( "AWS credentials not found. Ensure AWS_ACCESS_KEY_ID and " "AWS_SECRET_ACCESS_KEY are set, or use IAM role." ) from exc except ClientError as exc: error_code = exc.response.get("Error", {}).get("Code", "Unknown") if error_code == "404": raise S3ConfigurationError( f"S3 bucket '{self.config.bucket}' does not exist." ) from exc elif error_code == "403": raise S3ConfigurationError( f"Access denied to S3 bucket '{self.config.bucket}'. " "Check IAM permissions." ) from exc else: raise S3ConfigurationError( f"Failed to access S3 bucket '{self.config.bucket}': " f"{error_code} - {exc}" ) from exc def close(self) -> None: """Close S3 client connection.""" # boto3 clients don't require explicit closing, but we reset state self._s3_client = None self._connected = False logger.info("S3 connection closed.") def _s3_key_for(self, local_path: Path) -> str: """Construct S3 key for a local file. Uses only the basename to prevent path traversal. Args: local_path: Local file path. Returns: S3 key with folder prefix and filename. """ fname = local_path.name if self.config.sanitized_folder: return f"{self.config.sanitized_folder}/{fname}" return fname def upload_file(self, local_path: Path) -> str: """Upload a single file to S3 with retry logic. Args: local_path: Path to the local file. Returns: S3 key of the uploaded file. Raises: S3UploadError: If upload fails after all retries. S3Error: If client is not connected. """ if not self._connected or not self._s3_client: raise S3Error("S3 client not connected") s3_key = self._s3_key_for(local_path) attempts = self.config.retries + 1 for attempt in range(1, attempts + 1): try: logger.debug( "Uploading '%s' -> 's3://%s/%s' (attempt %d/%d)", local_path, self.config.bucket, s3_key, attempt, attempts, ) self._s3_client.upload_file( str(local_path), self.config.bucket, s3_key, ) logger.info( "Uploaded file '%s' to s3://%s/%s", local_path.name, self.config.bucket, s3_key, ) return s3_key except Exception as exc: if attempt == attempts: logger.error( "Failed to upload '%s' to S3: %s", local_path.name, exc, ) raise S3UploadError( f"Failed to upload {local_path} to S3: {exc}" ) from exc backoff = self.config.backoff_sec * attempt logger.warning( "Attempt %d for '%s' failed (%s). Retry in %.1fs", attempt, local_path.name, exc, backoff, ) time.sleep(backoff) # Should not reach here, but satisfy type checker raise S3UploadError( # pragma: no cover f"Failed to upload {local_path}" ) def upload_files(self, files: Sequence[Path]) -> S3UploadResult: """Upload multiple files to S3. Args: files: Sequence of local file paths to upload. Returns: S3UploadResult with upload statistics. """ failed: List[Path] = [] uploaded = 0 total = len(files) s3_keys: List[str] = [] logger.info("Beginning S3 upload batch: %d files", total) for p in files: try: s3_key = self.upload_file(p) uploaded += 1 s3_keys.append(s3_key) except S3UploadError: failed.append(p) if not self.config.continue_on_error: logger.error( "Aborting batch after failure of '%s'", p.name ) break logger.info( "S3 upload batch complete: uploaded=%d failed=%d total=%d", uploaded, len(failed), total, ) return S3UploadResult( uploaded=uploaded, failed=failed, total=total, s3_keys=s3_keys, ) class _MockS3Client: """Mock S3 client for testing without real AWS connections.""" def __init__(self) -> None: """Initialize mock client with empty upload tracking.""" self.uploads: List[tuple] = [] def head_bucket(self, Bucket: str) -> dict: # noqa: N803 """Simulate head_bucket call.""" logger.debug("(mock) head_bucket for '%s'", Bucket) return {} def upload_file( self, local_path: str, bucket: str, key: str ) -> None: """Simulate file upload.""" self.uploads.append((local_path, bucket, key)) logger.debug( "(mock) upload_file '%s' -> s3://%s/%s", local_path, bucket, key, ) def transfer_to_s3_if_enabled( file_paths: Iterable[str], config: S3Config, ) -> Optional[S3UploadResult]: """Convenience function to perform conditional S3 transfer. Args: file_paths: Iterable of local file path strings. config: S3Config instance. Returns: S3UploadResult if transfer executed else None. """ file_list = list(file_paths) if not file_list: logger.info("No files provided for S3 transfer; skipping.") return None mgr = S3TransferManager(config) mgr.connect() try: path_objs = [Path(p) for p in file_list] return mgr.upload_files(path_objs) finally: mgr.close()