"""Compression utilities for feed file processing. This module provides gzip compression functionality for SAP Settlement feed files. Files are compressed before S3 transfer per INT-2486 requirements. Guiding principles: - Robust error handling with clear logging - Never log file contents, only metadata (paths, sizes) - Thread-safe operations using standard library gzip module - Compression ratios logged for observability """ from __future__ import annotations import gzip import logging import os import shutil from pathlib import Path from typing import List, Optional, Tuple __all__ = [ "gzip_file", "gzip_files_in_directory", "ungzip_file", "CompressionResult", "CompressionError", ] logger = logging.getLogger( os.environ.get("LOGGER_NAME", "sme-feed-file-exporter") ) class CompressionError(Exception): """Raised when compression or decompression fails.""" class CompressionResult: """Result of a compression operation. Attributes: source_path: Original file path. output_path: Compressed file path. original_size: Size of original file in bytes. compressed_size: Size of compressed file in bytes. compression_ratio: Ratio of compression (compressed/original). """ def __init__( self, source_path: Path, output_path: Path, original_size: int, compressed_size: int, ): """Initialize compression result. Args: source_path: Path to the original source file. output_path: Path to the compressed output file. original_size: Size of original file in bytes. compressed_size: Size of compressed file in bytes. """ self.source_path = source_path self.output_path = output_path self.original_size = original_size self.compressed_size = compressed_size @property def compression_ratio(self) -> float: """Return compression ratio (compressed/original). Returns 1.0 if original size is 0 to avoid division by zero. """ if self.original_size == 0: return 1.0 return self.compressed_size / self.original_size @property def savings_percent(self) -> float: """Return percentage of space saved by compression.""" return (1.0 - self.compression_ratio) * 100 def __repr__(self) -> str: """Return string representation of compression result.""" return ( f"CompressionResult(source={self.source_path.name}, " f"output={self.output_path.name}, " f"ratio={self.compression_ratio:.2%})" ) def gzip_file( source_path: str | Path, output_path: Optional[str | Path] = None, delete_original: bool = False, compression_level: int = 9, ) -> CompressionResult: """Compress a single file using gzip. Args: source_path: Path to the file to compress. output_path: Path for the compressed file. If None, appends '.gz' to the source path. delete_original: If True, delete the original file after successful compression. compression_level: Gzip compression level (1-9, default 9 = maximum). Returns: CompressionResult with compression statistics. Raises: CompressionError: If compression fails. FileNotFoundError: If source file does not exist. """ source = Path(source_path) if not source.exists(): raise FileNotFoundError(f"Source file not found: {source}") if not source.is_file(): raise CompressionError(f"Source path is not a file: {source}") # Default output path: append .gz if output_path is None: output = source.with_suffix(source.suffix + ".gz") else: output = Path(output_path) # Ensure output directory exists output.parent.mkdir(parents=True, exist_ok=True) original_size = source.stat().st_size logger.debug( "Compressing file '%s' (%d bytes) to '%s'", source.name, original_size, output.name, ) try: with open(source, "rb") as f_in: with gzip.open( output, "wb", compresslevel=compression_level ) as f_out: shutil.copyfileobj(f_in, f_out) compressed_size = output.stat().st_size result = CompressionResult( source_path=source, output_path=output, original_size=original_size, compressed_size=compressed_size, ) logger.info( "Compressed '%s': %d -> %d bytes (%.1f%% savings)", source.name, original_size, compressed_size, result.savings_percent, ) if delete_original: source.unlink() logger.debug("Deleted original file: %s", source.name) return result except Exception as exc: # Clean up partial output file if it exists if output.exists(): try: output.unlink() except Exception: # pragma: no cover pass raise CompressionError( f"Failed to compress {source}: {exc}" ) from exc def gzip_files_in_directory( directory: str | Path, pattern: str = "*.TXT", delete_originals: bool = False, compression_level: int = 9, ) -> Tuple[List[CompressionResult], List[Tuple[Path, Exception]]]: """Compress all matching files in a directory. Args: directory: Path to the directory containing files to compress. pattern: Glob pattern to match files (default: "*.TXT"). delete_originals: If True, delete original files after compression. compression_level: Gzip compression level (1-9). Returns: Tuple of (successful_results, failed_files) where: - successful_results: List of CompressionResult for successful ops - failed_files: List of (Path, Exception) tuples for failures Raises: FileNotFoundError: If directory does not exist. """ dir_path = Path(directory) if not dir_path.exists(): raise FileNotFoundError(f"Directory not found: {dir_path}") if not dir_path.is_dir(): raise CompressionError(f"Path is not a directory: {dir_path}") files = list(dir_path.glob(pattern)) logger.info( "Found %d files matching pattern '%s' in '%s'", len(files), pattern, dir_path, ) successful: List[CompressionResult] = [] failed: List[Tuple[Path, Exception]] = [] for file_path in files: try: result = gzip_file( file_path, delete_original=delete_originals, compression_level=compression_level, ) successful.append(result) except Exception as exc: logger.error("Failed to compress '%s': %s", file_path.name, exc) failed.append((file_path, exc)) logger.info( "Compression complete: %d succeeded, %d failed", len(successful), len(failed), ) return successful, failed def ungzip_file( source_path: str | Path, output_path: Optional[str | Path] = None, delete_original: bool = False, ) -> Path: """Decompress a gzip file. Primarily for testing and debugging purposes. Args: source_path: Path to the .gz file to decompress. output_path: Path for the decompressed file. If None, removes the '.gz' extension from the source path. delete_original: If True, delete the .gz file after decompression. Returns: Path to the decompressed file. Raises: CompressionError: If decompression fails. FileNotFoundError: If source file does not exist. """ source = Path(source_path) if not source.exists(): raise FileNotFoundError(f"Source file not found: {source}") # Default output path: remove .gz suffix if output_path is None: if source.suffix.lower() == ".gz": output = source.with_suffix("") else: output = source.with_suffix(".decompressed") else: output = Path(output_path) # Ensure output directory exists output.parent.mkdir(parents=True, exist_ok=True) logger.debug("Decompressing '%s' to '%s'", source.name, output.name) try: with gzip.open(source, "rb") as f_in: with open(output, "wb") as f_out: shutil.copyfileobj(f_in, f_out) logger.info("Decompressed '%s' -> '%s'", source.name, output.name) if delete_original: source.unlink() logger.debug("Deleted compressed file: %s", source.name) return output except Exception as exc: # Clean up partial output file if it exists if output.exists(): try: output.unlink() except Exception: # pragma: no cover pass raise CompressionError( f"Failed to decompress {source}: {exc}" ) from exc