"""Singleton resource management for shared thread pools and connections. Provides centralized access to reusable resources with lazy initialization and automatic cleanup. Uses double-checked locking for thread-safe singleton instantiation in Lambda execution contexts. Resources: - CPU pool: Thread pool for compute-intensive tasks (DuckDB, validation) - IO pool: Thread pool for network operations (Snowflake, S3) - S3 connection: Shared boto3 S3 client """ import atexit from concurrent.futures import ThreadPoolExecutor from threading import Lock from src.connectors.s3 import S3Connection, get_s3_connection from src.utils.file_utils import ( get_num_cpu_workers, get_num_io_workers, ) class ResourceManager: """Manages singleton instances of thread pools and S3 connections. Provides lazy-initialized, thread-safe access to shared resources using double-checked locking pattern. Resources are automatically cleaned up at program exit. """ _cpu_pool: ThreadPoolExecutor | None = None _io_pool: ThreadPoolExecutor | None = None _lock = Lock() _s3_connection: S3Connection | None = None @classmethod def get_cpu_pool(cls) -> ThreadPoolExecutor: """For heavy processing (e.g. DuckDB, data cleaning).""" if cls._cpu_pool is None: with cls._lock: if cls._cpu_pool is None: # CPU should be tight: ~1x vCPU count workers = int(get_num_cpu_workers()) cls._cpu_pool = ThreadPoolExecutor( max_workers=workers, thread_name_prefix='cpu_' ) atexit.register(cls._cpu_pool.shutdown, wait=False) return cls._cpu_pool @classmethod def get_io_pool(cls) -> ThreadPoolExecutor: """For networking (e.g., Snowflake fetches, S3 uploads).""" if cls._io_pool is None: with cls._lock: if cls._io_pool is None: # IO can be wide: 4-5x vCPU count workers = int(get_num_io_workers()) cls._io_pool = ThreadPoolExecutor( max_workers=workers, thread_name_prefix='io_' ) atexit.register(cls._io_pool.shutdown, wait=False) return cls._io_pool @classmethod def get_s3_connection(cls) -> S3Connection: """Get shared S3 connector for Lambda invocations.""" if cls._s3_connection is None: with cls._lock: if cls._s3_connection is None: cls._s3_connection = get_s3_connection() return cls._s3_connection