"""Config for YouTube Statistics Workflow.""" import os import subprocess from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import serialization from fangare import catchers YOUTUBE_LINK = 'https://www.youtube.com/channel/' HTTP_YOUTUBE_LINK = 'http://www.youtube.com/channel/' ENV = os.environ.get('Environment') FLOW_NAME = 'youtube-statistics' STATS_NAME = 'youtube_statistics' if os.environ.get('Environment') == 'prod': try: GIT_HASH_OF_LAST_COMMIT = ( subprocess.check_output( 'git rev-parse HEAD', shell=True).decode().strip()) except subprocess.CalledProcessError: GIT_HASH_OF_LAST_COMMIT = 'youtube_statistics_commit' else: GIT_HASH_OF_LAST_COMMIT = 'youtube_statistics_dev_commit' # get Snowflake private_key SNOWFLAKE_PRIVATE_KEY_PATH = os.getenv( 'SNOWFLAKE_PRIVATE_KEY_PATH', '{}/.ssh/snowflake/rsa_key.p8'.format(os.getenv('HOME'))) SNOWFLAKE_KEY_PASSPHRASE = os.getenv('SNOWFLAKE_KEY_PASSPHRASE') if SNOWFLAKE_KEY_PASSPHRASE: # Snowflake key with open(SNOWFLAKE_PRIVATE_KEY_PATH, 'rb') as key: p_key = serialization.load_pem_private_key( key.read(), password=SNOWFLAKE_KEY_PASSPHRASE.encode(), backend=default_backend() ) private_key = p_key.private_bytes( encoding=serialization.Encoding.DER, format=serialization.PrivateFormat.PKCS8, encryption_algorithm=serialization.NoEncryption()) else: private_key = None SF_CONFIG = { 'role': os.environ.get('SNOWFLAKE_ROLE'), 'warehouse': os.environ.get('SNOWFLAKE_WAREHOUSE'), 'db': os.environ.get('SNOWFLAKE_DATABASE'), 'schema': os.environ.get('SNOWFLAKE_SCHEMA'), 'user': os.environ.get('SNOWFLAKE_USER'), 'password': catchers.get_secret('SNOWFLAKE_PASSWORD', 'youtube-statistics', ENV), 'account': os.environ.get('SNOWFLAKE_ACCOUNT'), 'private_key': private_key } aws = { 'aws_key_id': os.environ.get('AWS_ACCESS_KEY_ID'), 'aws_secret_key': os.environ.get('AWS_SECRET_ACCESS_KEY') } # a DB table postfix for specifying country, team etc. (e.g. '_UK') postfix = os.environ.get('SNOWFLAKE_DATABASE_POSTFIX', '') snowflake_table_names = { 'youtube_artist_to_track': 'artist_to_track' + postfix, 'youtube_statistics': 'youtube_statistics' + postfix, 'staging_raw': 'staging_raw_youtube_statistics' + postfix } youtube = { 'username': os.environ.get('YOUTUBE_USERNAME'), 'password': os.environ.get('YOUTUBE_PASSWORD'), }