"""Non ETL specific s3 utility functions.""" import csv import boto3 def split_url(url): """Split an s3 url to it's parts. Supports https:// and s3:// formats. Args: url (str): s3 object url. Returns: dict: labeled parts of the url. """ schema, path = url.split('://', 1) # discard the s3.amazonaws.com domain if it's an https:// address if schema == 'https': path = path.split('/', 1)[1] segments = path.split('/', 1) return {'bucket': segments[0], 'key': segments[1]} def get_object(url): """Get the s3 object from a full url. Args: url (str): s3 object url. Returns: boto.S3.Object: S3 key object. """ s3_parts = split_url(url) s3 = boto3.resource('s3') return s3.Object(s3_parts['bucket'], s3_parts['key']) def download_csv(s3_url): """Download CSV file from S3 bucket. Args: s3_url (str): S3 url. Returns: tuple: header and list of rows. """ data = get_object(s3_url).get()['Body'].read() lines = data.decode().splitlines() _, *rows = csv.reader(lines) return rows