"""Helper functions for workflows.""" import base64 import os import boto3 from botocore.exceptions import ClientError from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import serialization from garcon_contrib.aws.utils import garcon_s3 import jsonschema from secrets_manager.swf_ext import SWFSecretsManager from feed_ingestion.conf import config from feed_ingestion.flows import secrets_manager_flows from feed_ingestion.util.sentry_util import send_error_or_warning def get_filesize(destination_bucket_name, path, expected_bucket_owner='437795906767'): """ Calculate filesize of file stored on S3. Args: conn (S3Connection): Connection for connecting to S3. destination_bucket_name (str): S3 bucket name. path (str): Path to S3 file. expected_bucket_owner (str): S3 bucket owner. Returns: file_size (int): Filesize of the file on S3. """ s3_client = boto3.client('s3') s3_client.get_object( Bucket=destination_bucket_name, Key=path, ExpectedBucketOwner=expected_bucket_owner ) return s3_client.head_object( Bucket=destination_bucket_name, Key=path, ExpectedBucketOwner=expected_bucket_owner)['ContentLength'] def check_s3_key_exist(s3_key_path, expected_bucket_owner='437795906767'): """Check if S3 object exists in the specific path. Args: s3_key_path (str): Full path to the S3 key. expected_bucket_owner (str): S3 bucket owner. Returns: bool: True or False of whether it exist. """ s3_client = boto3.client('s3') bucket_name, key_path = garcon_s3.extract_bucket_path(s3_key_path) try: s3_client.head_object( Bucket=bucket_name, Key=key_path, ExpectedBucketOwner=expected_bucket_owner ) return True except ClientError as e: if e.response['Error']['Code'] == '404': return False raise e def download(s3_path, local_path, expected_bucket_owner='437795906767'): """Download S3 object(s) to local directory. Args: s3_path (str): Prefix of S3 object(s). Eg. s3://bucket/file - will get s3://bucket/file_part1.gz, s3://bucket/file_part2.gz..... local_path (str): Local directory. expected_bucket_owner (str): S3 bucket owner. """ s3_client = boto3.client('s3') bucket, bucket_path = garcon_s3.extract_bucket_path(s3_path) s3_client.download_file( bucket, bucket_path, local_path, ExtraArgs={'ExpectedBucketOwner': expected_bucket_owner} ) def upload_raw_file_to_s3( source_path, s3_path, expected_bucket_owner='437795906767'): """Upload local file to S3. Args: source_path (str): Full path to local path. s3_path (str): Full S3 destination path. expected_bucket_owner (str): S3 bucket owner. """ s3_client = boto3.client('s3') bucket, path = garcon_s3.extract_bucket_path(s3_path) s3_client.upload_file( source_path, bucket, path, ExtraArgs={'ExpectedBucketOwner': expected_bucket_owner} ) def validate_header(row, fieldnames): """Validate header row of a CSV file. Args: row (list): First row (header) of a CSV file. fieldnames (list): Fieldnames extracted from schema. Raises: ValueError: When schema invalidation is found. """ for i, row_name in enumerate(row): if row_name != fieldnames[i]: raise ValueError( '{row_name} is invalid according to schema'.format( row_name=row_name)) def validate_row(row_schema, row): """Validate data row of a CSV file. Args: row_schema (dict): JSON schema of a row. row (dict): Row {"column_name": "value", ...} of a CSV file. Raises: jsonschema.exceptions.ValidationError: When schema error is found. """ jsonschema.Draft4Validator(row_schema).validate(row) def handle_download_error(status, err, raise_error): """Handle download error. Returns `{'stop': True}` if file is not available, reraises an exception and send notification in sentry. Args: status (int): Error status. err (Exception): Error. raise_error(bool): if True raise an exception, else send in Sentry. Returns: dict: stop response """ if status != 404: if raise_error: raise err else: send_error_or_warning(err) return {'stop': True} def handle_s3_download_error(err, raise_error=True): """Handle S3 download error. Returns `{'stop': True}` if file is not available, reraises an exception or send in Sentry otherwise. Args: err (S3ResponseError): Error. raise_error(bool): if True raise an exception, else send in Sentry. Returns: dict: stop response """ return handle_download_error(err.status, err, raise_error) def handle_http_download_error(err, raise_error=True): """Handle S3 download error. Returns `{'stop': True}` if file is not available, reraises an exception or send in Sentry otherwise. Args: err (HTTPError): Error. raise_error(bool): if True raise an exception, else send in Sentry. Returns: dict: stop response """ return handle_download_error(err.response.status_code, err, raise_error) def get_secret(service_name, secret_name): """Get secret value from SWFSecretsManager. For dev environment it loads value from environ instead of secrets manager. Args: service_name: service name defined in terraform configuration secret_name: key name for the secret Returns: secrets value or None if secrets not created """ secrets_manager_client = SWFSecretsManager( environment=config.ENV, service_name=service_name) try: return secrets_manager_client.get_cred(secret_name) except ClientError as e: # The secret is empty if e.response['Error']['Code'] == 'ResourceNotFoundException': return None else: raise e def get_sf_config(secrets_path): """Return Snowflake config with credentials. Args: secrets_path (str): Secrets manager path of the flow. Returns: dict: SF credentials """ if secrets_path not in secrets_manager_flows: return config.SF_CONFIG private_key = None private_key_string = get_secret(secrets_path, 'SNOWFLAKE_KEY') if private_key_string: private_key = decode_snowflake_key(private_key_string) if not private_key and config.ENV == 'dev': # Default for local dev snowflake_private_key_path = os.environ['HOME'] + \ '/.ssh/snowflake/rsa_key.p8' snowflake_private_key_path = os.environ.get( 'SNOWFLAKE_PRIVATE_KEY_PATH', snowflake_private_key_path) snowflake_key_passphrase = os.environ.get( 'SNOWFLAKE_KEY_PASSPHRASE', None) if snowflake_key_passphrase: with open(snowflake_private_key_path, 'rb') as key: p_key = serialization.load_pem_private_key( key.read(), password=snowflake_key_passphrase.encode(), backend=default_backend() ) private_key = p_key.private_bytes( encoding=serialization.Encoding.DER, format=serialization.PrivateFormat.PKCS8, encryption_algorithm=serialization.NoEncryption()) config.SF_CONFIG['private_key'] = private_key return config.SF_CONFIG def get_neo4j_config(flow_name): """Return Neo4j config with credentials. Args: flow_name (str): The name of the feed. Returns: dict: The Neo4j config. """ if flow_name not in secrets_manager_flows: return config.NEO4J_CONFIG secrets_manager_client = SWFSecretsManager( environment=config.ENV, service_name=flow_name) try: neo4j_user = secrets_manager_client.get_cred('NEO4J_USER') neo4j_password = secrets_manager_client.get_cred('NEO4J_PASSWORD') except ClientError as e: if e.response['Error']['Code'] == 'ResourceNotFoundException': pass else: raise e config.NEO4J_CONFIG['user'] = neo4j_user config.NEO4J_CONFIG['password'] = neo4j_password return config.NEO4J_CONFIG def decode_snowflake_key(snowflake_key): """Decode the Snowflake key.""" if not snowflake_key: return decoded_key = base64.b64decode( bytes(snowflake_key, encoding='utf-8')) return decoded_key