"""Common config for all the workflows.""" import base64 import logging import os from botocore.config import Config from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import serialization from garcon_contrib.dynamo_feed_status import config as garcon_config logger = logging.getLogger(__name__) SNOWFLAKE_PRIVATE_KEY_PATH = os.environ.get('SNOWFLAKE_PRIVATE_KEY_PATH', '') SNOWFLAKE_KEY_PASSPHRASE = os.environ.get('SNOWFLAKE_KEY_PASSPHRASE', '') ENV = os.environ.get('Environment') or 'test' if ENV == 'dev' and SNOWFLAKE_KEY_PASSPHRASE: # Snowflake key if not SNOWFLAKE_PRIVATE_KEY_PATH: SNOWFLAKE_PRIVATE_KEY_PATH = '{}/.ssh/snowflake/rsa_key.p8'.format( os.environ['HOME']) with open(SNOWFLAKE_PRIVATE_KEY_PATH, 'rb') as key: p_key = serialization.load_pem_private_key( key.read(), password=SNOWFLAKE_KEY_PASSPHRASE.encode(), backend=default_backend() ) private_key = p_key.private_bytes( encoding=serialization.Encoding.DER, format=serialization.PrivateFormat.PKCS8, encryption_algorithm=serialization.NoEncryption()) else: private_key_str = os.environ.get('SNOWFLAKE_KEY') if private_key_str: private_key = base64.b64decode(private_key_str.encode('utf-8')) else: private_key = None FILE_FORMAT_ARCHIVE_FROM_SNOWFLAKE = r""" FIELD_DELIMITER='\t' RECORD_DELIMITER='\n' COMPRESSION='GZIP' FIELD_OPTIONALLY_ENCLOSED_BY='"' TRIM_SPACE=TRUE""" def merge_configs(c1, c2): """Merge two flat configs. Values from c1 get overridden by values from c2 if the keys collide. Args: c1 (dict): First config. c2 (dict): Second config. Returns: dict: Result dict containing merged result. """ c1 = c1 or {} c2 = c2 or {} return {**c1, **c2} # Default Snowflake connection parameters excluding credentials SF_PARAMS = { 'role': os.environ.get('SNOWFLAKE_ROLE'), 'warehouse': os.environ.get('SNOWFLAKE_WAREHOUSE'), 'db': os.environ.get('SNOWFLAKE_DATABASE'), 'schema': os.environ.get('SNOWFLAKE_SCHEMA'), 'ocsp_fail_open': False # enforce OCSP checking } AWS_REGION = os.environ.get('AWS_REGION', os.environ.get( 'AWS_DEFAULT_REGION', 'us-east-1')) # common boto3 client config BOTO3_CONFIG = Config( region_name=AWS_REGION, ) DEFAULT_EXPECTED_BUCKET_OWNER = ('103233932089' if ENV in {'dev', 'local'} else '437795906767') EXPECTED_BUCKET_OWNER = (os.environ.get('EXPECTED_BUCKET_OWNER') or DEFAULT_EXPECTED_BUCKET_OWNER) # Snowflake connection credentials SF_CREDENTIALS = { 'user': os.environ.get('SNOWFLAKE_USER'), 'password': os.environ.get('SNOWFLAKE_PASSWORD'), 'account': os.environ.get('SNOWFLAKE_ACCOUNT'), 'private_key': private_key } SF_CONFIG = merge_configs(SF_PARAMS, SF_CREDENTIALS) NEO4J_CONFIG = { 'url': os.environ.get('NEO4J_URL'), 'user': os.environ.get('NEO4J_USER'), 'password': os.environ.get('NEO4J_PASSWORD'), 'database': os.environ.get('NEO4J_DATABASE_NAME', 'graph.db'), } # autoconfig feed_status table if not set explicitly if not garcon_config.feed_ingestion_table and ENV: garcon_config.feed_ingestion_table = f'{ENV}_feed_ingestion_status' logger.info(f'Autoconfigured dynamo_feed_status table ' f'to {garcon_config.feed_ingestion_table}')