"""Config.""" from os import environ from airflow.models import Variable import httpx from owsclient import M2MTokenManager, OwsClient from lib.constants import SERVICE_NAME from lib.utils.secrets_manager import SecretsManager def environ_get(variable_name, default_value=None): """Get environment variable from airflow first. Example for Managed Airflow configuration options: custom.ows_env converts to AIRFLOW__CUSTOM__OWS_ENV """ airflow_name = f'AIRFLOW__CUSTOM__{variable_name}' airflow_value = environ.get(airflow_name) if airflow_value: return airflow_value return environ.get(variable_name, default_value) DEV_OWS_ENV = 'dev' QA_OWS_ENV = 'qa' PROD_OWS_ENV = 'prod' OWS_ENV = environ_get('OWS_ENV', DEV_OWS_ENV) # S3 buckets S3_ACCOUNT_ID = environ_get('S3_ACCOUNT_ID', '437795906767') S3_PAYMENTS_BUCKET_NAME = environ_get('S3_PAYMENTS_BUCKET_NAME', 'qa-abacus-payments') S3_SALES_BUCKET_NAME = environ_get('S3_SALES_BUCKET_NAME') S3_STATIC_DATA_BUCKET_NAME = environ_get('S3_STATIC_DATA_BUCKET_NAME') # Lambdas ADJUSTMENTS_APPLY_LAMBDA_NAME = environ_get( 'ADJUSTMENTS_APPLY_LAMBDA_NAME', f'{OWS_ENV}-lambda-abacus-adjustments-apply' ) AV_SCAN_LAMBDA_NAME = environ_get( 'AV_SCAN_LAMBDA_NAME', f'{OWS_ENV}-lambda-av-scan-containerized' ) PAYMENTS_GENERATE_PROCESSOR_LAMBDA_NAME = environ_get( 'PAYMENTS_GENERATE_PROCESSOR_LAMBDA_NAME', f'{OWS_ENV}-lambda-abacus-payments-generate' ) RESERVES_TAKE_LAMBDA_NAME = environ_get( 'RESERVES_TAKE_LAMBDA_NAME', f'{OWS_ENV}-lambda-abacus-reserves-take' ) RESERVES_SCHEDULE_LAMBDA_NAME = environ_get( 'RESERVES_SCHEDULE_LAMBDA_NAME', f'{OWS_ENV}-lambda-abacus-reserves-schedule' ) ROYALTY_PROCESSOR_LAMBDA_NAME = environ_get( 'ROYALTY_PROCESSOR_LAMBDA_NAME', f'{OWS_ENV}-lambda-abacus-commit-royalties' ) COMMIT_VAT_LAMBDA_NAME = environ_get( 'COMMIT_VAT_LAMBDA_NAME', f'{OWS_ENV}-lambda-abacus-commit-vat' ) LEDGER_ACCOUNTING_RUN_BALANCE_LAMBDA_NAME = environ_get( 'LEDGER_ACCOUNTING_RUN_BALANCE_LAMBDA_NAME', f'{OWS_ENV}-lambda-abacus-ledger-accounting-run-balance' ) VAT_CALCULATE_LAMBDA_NAME = environ_get( 'VAT_CALCULATE_LAMBDA_NAME', f'{OWS_ENV}-lambda-abacus-vat-calculate' ) VAT_EXEMPT_LAMBDA_NAME = environ_get( 'VAT_EXEMPT_LAMBDA_NAME', f'{OWS_ENV}-lambda-abacus-vat-exempt' ) PAYONEER_PAYMENTS_PAYOUT_LAMBDA_NAME = environ_get( 'PAYONEER_PAYMENT_PAYOUT_LAMBDA_NAME', f'{OWS_ENV}-lambda-abacus-payoneer-payments-payout' ) RESERVES_RELEASE_LAMBDA_NAME = environ_get( 'RESERVES_RELEASE_LAMBDA_NAME', f'{OWS_ENV}-lambda-abacus-reserves-release' ) ADJUSTMENT_FILE_VALIDATION_LAMBDA_NAME = environ_get( 'ADJUSTMENT_FILE_VALIDATION_LAMBDA_NAME', f'{OWS_ENV}-lambda-abacus-adjustment-file-validation' ) ADJUSTMENT_FILE_IMPORT_LAMBDA_NAME = environ_get( 'ADJUSTMENT_FILE_IMPORT_LAMBDA_NAME', f'{OWS_ENV}-lambda-abacus-adjustment-file-import' ) COMMIT_MECHANICALS_LAMBDA_NAME = environ_get( 'COMMIT_MECHANICALS_LAMBDA_NAME', f'{OWS_ENV}-lambda-abacus-commit-mechanicals' ) GENERATE_FLOWTHROUGH_ADJUSTMENTS_LAMBDA_NAME = environ_get( 'GENERATE_FLOWTHROUGH_ADJUSTMENTS_LAMBDA_NAME', f'{OWS_ENV}-lambda-abacus-generate-flowthrough-adjustments' ) ADJUSTMENTS_JSON_VALIDATION_LAMBDA_NAME = environ_get( 'ADJUSTMENTS_JSON_VALIDATION_LAMBDA_NAME', f'{OWS_ENV}-lambda-abacus-adjustments-json-validation' ) ADJUSTMENTS_JSON_IMPORT_LAMBDA_NAME = environ_get( 'ADJUSTMENTS_JSON_IMPORT_LAMBDA_NAME', f'{OWS_ENV}-lambda-abacus-adjustments-json-import' ) # ECS tasks EXTRACT_SALES_ECS_SERVICE_NAME = environ_get( 'EXTRACT_SALES_ECS_SERVICE_NAME', f'{OWS_ENV}-ecs-abacus-extract-sales' ) EXTRACT_SALES_DAG_EXECUTION_ROLE = environ_get( 'EXTRACT_SALES_DAG_EXECUTION_ROLE', None ) # Art Relations AR_MYSQL_DB_HOST = environ_get('AR_MYSQL_DB_HOST', 'qa.db.qaorch.com') AR_MYSQL_DB_USER = environ_get('AR_MYSQL_DB_USER') AR_MYSQL_DB_PASS = environ_get('AR_MYSQL_DB_PASS') AR_MYSQL_CONN_NAME = environ_get( 'ART_RELATIONS_CONNECTION_NAME', 'ART_RELATIONS_CONNECTION_URI' ) # Snowflake SNOWFLAKE_ORCH_APP_REPORTING_SCHEMA = environ_get( 'SNOWFLAKE_ORCH_APP_REPORTING_SCHEMA', 'ART_RELATIONS_PROD_ART_RELATIONS' ) SNOWFLAKE_CONN_NAME = environ_get( 'SNOWFLAKE_CONNECTION_NAME', 'ORCH_SNOWFLAKE_DEFAULT' ) # NOTE: This is only used for QA and Production. # The private key and the passphrase are stored in Secrets Manager. # We retrieve them through Airflow's Variable system. # For this to work a `secrets_backend` with the correct `variables_prefix` needs # to be configured. SNOWFLAKE_PRIVATE_KEY = Variable.get('SNOWFLAKE_PRIVATE_KEY', None) SNOWFLAKE_KEY_PASSPHRASE = Variable.get('SNOWFLAKE_KEY_PASSPHRASE', None) FACTS_SCHEMA = environ_get('FACTS_SCHEMA', 'qa') ABACUS_PARQUET_STAGE = environ_get( 'ABACUS_PARQUET_STAGE', 'ABACUS_PARQUET_FILE' ) ABACUS_TSV_STAGE = environ_get( 'ABACUS_TSV_STAGE', 'ABACUS_TSV_FILE' ) SNAPSHOT_CONTRACTS_BATCH_SIZE = int( environ_get('SNAPSHOT_CONTRACTS_BATCH_SIZE', '500') ) # comma separated list of country codes VAT_APPLIED_COUNTRY_CODES = environ_get( 'VAT_APPLIED_COUNTRY_CODES', 'GBR' ) # DAG names (used to enable feature flags per DAG) ADJUSTMENT_FILE_GENERATE_DAG_NAME = environ_get( 'ADJUSTMENT_FILE_GENERATE_DAG_NAME', 'adjustment_file_generate_dag' ) # Instantiate re-usable OwsClient instance secrets_manager = SecretsManager() m2m_token_manager = M2MTokenManager( environment=OWS_ENV, service_name=SERVICE_NAME, secrets_manager=secrets_manager ) ows_client = OwsClient( environment=OWS_ENV, service_name=SERVICE_NAME, timeout=httpx.Timeout(60.0), m2m_token_manager=m2m_token_manager if OWS_ENV != DEV_OWS_ENV else None,)