import io import os import logging from time import sleep import boto3 from botocore.exceptions import ClientError from vector_utils.connections.connection import Connection from vector_utils.connections import exceptions class S3Connection(Connection): """The connection class is used for S3 connections. Attributes: conn_obj (ConnectionInfo) logger (obj): Provide a logging object connection (any, optional) """ def __init__(self, conn_obj, logger=None): super().__init__(conn_obj) self.connection = None self._s3client = None self._logger = logger or self._setup_logger() self.login() @property def logger(self): return self._logger @logger.setter def logger(self, value): self._logger = value def _setup_logger(self): """Setup logger helper.""" logger = logging.getLogger(__name__) logger.addHandler(logging.NullHandler()) return logger def login(self): """Login and setup connection to s3.""" self.connection = boto3.client( 's3', aws_access_key_id=self.conn_obj.user_name, aws_secret_access_key=self.conn_obj.password, region_name=self.conn_obj.s3_region) @Connection.handle_exceptions def check_connection(self, remote_initial_dir): pass @Connection.handle_exceptions def close_connection(self): """Close the client connection.""" pass @Connection.handle_exceptions def delete(self, filename, remove_dir=False): pass @Connection.handle_exceptions def file_exists(self, file_name): """Check file exists. Args: file_name (str): A valid filename path Returns: bool """ try: self.connection.head_object( Bucket=self.conn_obj.domain_name, Key=file_name ) return True except: return False @Connection.handle_exceptions def file_size(self, file_name): """Return file size. Args: file_name (str): Full file path Returns: int: The size in bytes. """ try: data = self.connection.head_object( Bucket=self.conn_obj.domain_name, Key=file_name ) return data['ContentLength'] except ClientError: raise FileNotFoundError('No such file') @Connection.handle_exceptions def is_dir(self, dirname): """Change directory. Args: dirname (str): The directory to change to. """ pass @Connection.handle_exceptions def mkdir(self, dest_dir): """Create directories recursively. Args: dest_dir (str): The full directory path. Returns: bool: True if we were able to create the path """ data = self.scan_dir(dest_dir) if not data: data = io.BytesIO(b'') self.connection.upload_fileobj( data, self.conn_obj.domain_name, dest_dir) return True return @Connection.handle_exceptions def rmdir(self, dir_name): """Remove the directory path. Args: dir_name (str): The full directory path. """ pass @Connection.handle_exceptions def reconnect(self): """Reconnect connection.""" self.login() @Connection.handle_exceptions def scan_dir(self, directory, exceptions=[]): """Return list of directories or files. Args: directory (str): The path which you which to scan. exceptions (list): The files or folders to exclude. Returns: list(str) """ results = self.connection.list_objects_v2( Bucket=self.conn_obj.domain_name, Prefix=directory, ) directory_results = [] if results['KeyCount']: for x in results['Contents']: if x['Key'] in exceptions: continue if os.path.split(x['Key'])[0] == results['Prefix']: directory_results.append( os.path.basename(x['Key']).strip('/')) else: directory_results.append( os.path.dirname(x['Key']).strip('/')) return directory_results @Connection.handle_exceptions def transfer_files(self, file_list, transfer_mode='upload', batch_file_id=None, pid=None, overwrite=None, check_delivered_files=True, d_job=None): """Transfer files up or down. Args: file_list (list(dict)): A list of remote/local keypairs transfer_mode (str): upload or download batch_file_id: pid: overwrite: check_delivered_files: d_job: Returns: Raises: Exception: We are unable to transfer files. """ attempts = 1 while attempts <= 5: self.logger.info('Delivery attempt # {}.'.format(attempts)) try: for file in file_list: local_file = file['local'] remote_file = file['remote'] self.logger.info('Start transferring.') if transfer_mode == 'download': with open(local_file, 'wb') as data: self.connection.download_fileobj( self.conn_obj.domain_name, remote_file, data) else: with open(local_file, 'rb') as data: self.connection.upload_fileobj( data, self.conn_obj.domain_name, remote_file.lstrip('/')) self.logger.info('Transfer finished.') self.logger.info('Delivery succeeded.') return True except Exception as e: self.logger.error('Error transferring file: {}'.format(e)) attempts += 1 sleep(15) self.logger.info('Delivery failed.') raise exceptions.TransferException()