"""Connects to AWS S3.""" import boto3 # AWS from integration_scripts import config aws_keys = config.AWS_ACCESS_KEY_ID and config.AWS_SECRET_ACCESS_KEY test_env = config.ENVIRONMENT == config.TEST_ENVIRONMENT if not aws_keys and not test_env: raise ConnectionAbortedError('AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY ' 'are required for S3 access.') # -- Basic S3 ---------------------------------------------------------------- # Create s3 object s3_resource = boto3.resource( 's3', aws_access_key_id=config.AWS_ACCESS_KEY_ID, aws_secret_access_key=config.AWS_SECRET_ACCESS_KEY) # Create s3 Client object s3_client = boto3.client( 's3', aws_access_key_id=config.AWS_ACCESS_KEY_ID, aws_secret_access_key=config.AWS_SECRET_ACCESS_KEY) # connect to buckets s3_bucket_name = config.S3_BUCKET # -- Dual (Source/Target) S3 ------------------------------------------------- if config.TARGET_AWS_ACCESS_KEY_ID and config.TARGET_AWS_SECRET_ACCESS_KEY: # Create s3 object target_s3_resource = boto3.resource( 's3', aws_access_key_id=config.TARGET_AWS_ACCESS_KEY_ID, aws_secret_access_key=config.TARGET_AWS_SECRET_ACCESS_KEY) # Create s3 Client object target_s3_client = boto3.client( 's3', aws_access_key_id=config.TARGET_AWS_ACCESS_KEY_ID, aws_secret_access_key=config.TARGET_AWS_SECRET_ACCESS_KEY) if config.TARGET_S3_BUCKET: # connect to buckets target_s3_bucket_name = config.TARGET_S3_BUCKET # s3 Config for download_fileobj() - chunks and threading S3_CONFIG = boto3.s3.transfer.TransferConfig( multipart_threshold=4000000, multipart_chunksize=4000000, use_threads=True)