"""Config file.""" from distutils.util import strtobool import os from constants.aws import S3_TARGET_ROLE_ARNS # LOAD_ENV = strtobool(str(os.environ.get('LOAD_ENV', 'True'))) # if LOAD_ENV: # from dotenv import load_dotenv # load_dotenv(override=True, verbose=True) # Sentry SENTRY = os.environ.get("SENTRY_DSN") or None # SNOWFLAKE SNOWFLAKE_SOURCE_TABLE = os.environ.get( "SNOWFLAKE_SOURCE_TABLE", "SONY_STARS_MONTHLY_FINANCIAL_FEED_VIEW" ) PERIOD_ID = os.environ.get("PERIOD_ID", None) # Files and Folders FILE_OUTPUT_PATH = os.environ.get("FILE_OUTPUT_PATH", "output/{date}") PROCESS_MODE = os.environ.get("PROCESS_MODE", "US") COPY_TABLE = strtobool(str(os.environ.get("COPY_TABLE", "True"))) DELETE_COPY = strtobool(str(os.environ.get("DELETE_COPY", "True"))) # ----------------------------- SFTP Transfer ----------------------------- # Whether to transfer non-zipped files via SFTP. # Reads SFTP_TRANSFER_ENABLED (Jenkins param name) first, then falls back to # TRANSFER_FILES (legacy CLI / original param name) for backwards compatibility. # Either truthy value enables transfer. SFTP_TRANSFER_ENABLED = strtobool( str( os.environ.get('SFTP_TRANSFER_ENABLED') or os.environ.get('TRANSFER_FILES', 'False') ) ) # Core connection parameters (default values per TICKET.md assumptions) SFTP_HOST = os.environ.get('SFTP_HOST', 'ftp.sme-dsr.com') SFTP_PORT = int(os.environ.get('SFTP_PORT', '22')) SFTP_USERNAME = os.environ.get('SFTP_USER', 'FTP_Orchard') SFTP_REMOTE_DIR = os.environ.get('SFTP_FOLDER', 'monthly') # Password should come from Jenkins Credentials / secret manager injected # as env var # TODO: DEPRECATE - Password auth will be removed in future version. # Use SFTP_AUTH_MODE=key with SFTP_PRIVATE_KEY_PATH instead. SFTP_PASSWORD = os.environ.get('SFTP_PASSWORD') # NEVER log this # Authentication mode: 'password', 'key', or 'auto' # - 'key': Use private key only from SFTP_PRIVATE_KEY_PATH (DEFAULT) # - 'password': Use password only (DEPRECATED - must set explicitly) # - 'auto': Try key first, fallback to password if both available # TODO: DEPRECATE - Remove 'password' and 'auto' modes entirely. SFTP_AUTH_MODE = os.environ.get('SFTP_AUTH_MODE', 'key').lower() # Path to private key file for key-based authentication # Defaults to ~/.ssh/id_rsa if not specified SFTP_PRIVATE_KEY_PATH = os.environ.get( 'SFTP_PRIVATE_KEY_PATH', '~/.ssh/id_rsa' ) # Operational tuning SFTP_RETRIES = int(os.environ.get('SFTP_RETRIES', '3')) SFTP_RETRY_BACKOFF_SEC = float(os.environ.get('SFTP_RETRY_BACKOFF_SEC', '2')) SFTP_TIMEOUT_SEC = float(os.environ.get('SFTP_TIMEOUT_SEC', '30')) # Strict host key checking (default True). In dev you can disable by # setting to 'False' SFTP_STRICT_HOST_KEY = strtobool( str(os.environ.get('SFTP_STRICT_HOST_KEY', 'True')) ) # Continue on error instead of failing entire job (default False) SFTP_CONTINUE_ON_ERROR = strtobool( str(os.environ.get('SFTP_CONTINUE_ON_ERROR', 'False')) ) # ----------------------------- S3 Transfer --------------------------------- # There is ONE S3 transfer capability. It uses the environment's AWS # credentials (IAM role or env vars) to deliver files to a target bucket and # folder. The whitelist guard in feed_file_exporter.py determines which # PROCESS_MODE values are authorised to use it. S3_TRANSFER_ENABLED = strtobool( str(os.environ.get('S3_TRANSFER_ENABLED', 'False')) ) # Comma-separated list of PROCESS_MODE values authorised to use S3 transfer. # Default: SAP_SETTLEMENT only. Override to extend, e.g. 'SAP_SETTLEMENT,US'. S3_ALLOWED_MODES: frozenset[str] = frozenset( m.strip() for m in os.environ.get('S3_ALLOWED_MODES', 'SAP_SETTLEMENT').split(',') if m.strip() ) S3_GZIP_ENABLED = strtobool( str(os.environ.get('S3_GZIP_ENABLED', 'True')) ) S3_TARGET_ENV = os.environ.get('S3_TARGET_ENV', 'qa').strip().lower() if S3_TARGET_ENV not in S3_TARGET_ROLE_ARNS: S3_TARGET_ENV = 'qa' S3_ROLE_ARN = os.environ.get( 'S3_ROLE_ARN', S3_TARGET_ROLE_ARNS[S3_TARGET_ENV], ) # Transfer mode: 'Individual Files' | 'Zip File' | 'Both' | 'None' S3_TRANSFER_MODE = os.environ.get('S3_TRANSFER_MODE', 'Individual Files') S3_BUCKET = os.environ.get('S3_BUCKET', 'prod-abacus-sap') S3_FOLDER = os.environ.get('S3_FOLDER', 'bw_settlement_files') S3_REGION = os.environ.get('S3_REGION', 'us-east-1') # Operational tuning S3_RETRIES = int(os.environ.get('S3_RETRIES', '3')) S3_RETRY_BACKOFF_SEC = float(os.environ.get('S3_RETRY_BACKOFF_SEC', '2')) S3_TIMEOUT_SEC = float(os.environ.get('S3_TIMEOUT_SEC', '60')) S3_CONTINUE_ON_ERROR = strtobool( str(os.environ.get('S3_CONTINUE_ON_ERROR', 'False')) ) # Fetch Strategy Configuration # Modes: STREAMING (default), CHUNKED, CURSOR FETCH_MODE = os.environ.get("FETCH_MODE", "STREAMING") # Chunk size for chunked/cursor fetching CHUNK_SIZE = int(os.environ.get("CHUNK_SIZE", "50000")) # ---------------------- Output Write Optimization (INT-2638) ---------------- # Buffer size for batched file writes (number of rows to accumulate before # flushing to disk). Larger values reduce syscall overhead but use more memory. # Default: 10000 rows (balance between memory and syscall efficiency). WRITE_BUFFER_SIZE = int(os.environ.get("WRITE_BUFFER_SIZE", "10000")) # File I/O buffer size in bytes for large file operations. # Default: 8MB (8388608 bytes) for efficient large file writes. FILE_BUFFER_SIZE = int(os.environ.get("FILE_BUFFER_SIZE", "8388608"))