import argparse from datetime import datetime import logging import os import urllib.request from urllib.parse import urlparse import boto3 DEFAULT_CHUNK_SIZE = 15000 logger = logging.getLogger('main') handler = logging.StreamHandler() logger.addHandler(handler) logger.setLevel(logging.INFO) task_token = os.environ.get('SFN_TASK_TOKEN', '') def download(url: str, s3_location: str, chunk_size: int) -> None: logger.info('Start downloading from {} to {}'.format(url, s3_location)) with urllib.request.urlopen(url) as resp: logger.info('Response code: {}'.format(resp.getcode())) logger.info('Response headers: {}'.format(resp.info())) filename = datetime.now().strftime('%Y%m%d%H%M%S_') + extract_filename(url) logger.info('File name: {}'.format(filename)) if 200 <= resp.getcode() < 300: s3_client = boto3.client('s3') s3_client.upload_fileobj(resp, s3_location, filename) send_success('{"status":"success"}') else: logger.error('Request failed with http code: {}', resp.getcode()) send_failure('Unable to download', 'Http error {}'.format(resp.getcode())) logger.info('Upload complete') def send_success(output: str) -> None: if task_token: sfn_client = boto3.client('stepfunctions') sfn_client.send_task_success(taskToken=task_token, output=output) def send_failure(error: str, cause: str) -> None: if task_token: sfn_client = boto3.client('stepfunctions') sfn_client.send_task_failure(taskToken=task_token, error=error, cause=cause) def extract_filename(url: str) -> str: parsed = urlparse(url) return os.path.basename(parsed.path) def read_params() -> argparse.Namespace: """Read parameters from command line arguments""" parser = argparse.ArgumentParser() parser.add_argument("--url", type=str, help="url to download file from", required=True) parser.add_argument("--s3-bucket", type=str, help="source bucket", required=True) parser.add_argument( "--chunk-size", type=int, default=DEFAULT_CHUNK_SIZE, help="max size of a byte buffer used during decompression") args = parser.parse_args() return args def log_context() -> None: logger.info("Env: {}".format(os.environ))