import io import logging import os from time import sleep import paramiko from vector_utils.connections.connection import Connection from vector_utils.connections import exceptions class SftpConnection(Connection): """The connection class is used for SFTP connections. Attributes: conn_obj (ConnectionInfo) logger (obj): Provide a logging object connection (any, optional) """ def __init__(self, conn_obj, logger=None): super().__init__(conn_obj) self.connection = None self._sftpclient = None self._logger = logger or self._setup_logger() self.login() @property def logger(self): return self._logger @logger.setter def logger(self, value): self._logger = value def _setup_logger(self): """Setup logger helper.""" logger = logging.getLogger(__name__) logger.addHandler(logging.NullHandler()) return logger def login(self): """Login and setup connection to sftp server.""" try: client = paramiko.SSHClient() client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) if self.conn_obj.authenticate_type == 'password': client.connect( hostname=self.conn_obj.domain_name, port=self.conn_obj.port, username=self.conn_obj.user_name, password=self.conn_obj.password ) elif self.conn_obj.authenticate_type == 'public_key': client.connect( hostname=self.conn_obj.domain_name, port=self.conn_obj.port, username=self.conn_obj.user_name, key_filename=self.conn_obj.priv_key, disabled_algorithms=self.conn_obj.sftp_disabled_algorithms ) else: raise Exception(f'Unsupported authentication type {self.conn_obj.authenticate_type}') self.connection = client self._sftpclient = client.open_sftp() except (paramiko.SSHException, paramiko.ssh_exception.NoValidConnectionsError, EOFError): raise exceptions.SSHException() except TimeoutError: raise exceptions.ConnectionTimeout() @Connection.handle_exceptions def check_connection(self, remote_initial_dir): pass @Connection.handle_exceptions def close_connection(self): """Close the client connection.""" self.connection.close() @Connection.handle_exceptions def delete(self, filename, remove_dir=False): pass @Connection.handle_exceptions def file_exists(self, file_name): """Check file exists. Args: file_name (str): A valid filename path Returns: bool """ dirname, basename = os.path.split(file_name.rstrip('/')) dirs = self.scan_dir(dirname) if basename in dirs: return True return False @Connection.handle_exceptions def file_size(self, file_name): """Return file size. Args: file_name (str): Full file path Returns: int: The size in bytes. """ return self._sftpclient.stat(file_name).st_size @Connection.handle_exceptions def is_dir(self, dirname): """Change directory. Args: dirname (str): The directory to change to. """ self._sftpclient.chdir(dirname) @Connection.handle_exceptions def mkdir(self, dest_dir): """Create directories recursively. Args: dest_dir (str): The full directory path. Returns: bool: True if we were able to create the path """ if dest_dir == '/': self._sftpclient.chdir('/') return if dest_dir == '': return try: self._sftpclient.chdir(dest_dir) except (IOError, FileNotFoundError): dirname, basename = os.path.split(dest_dir.rstrip('/')) self.mkdir(dirname) self._sftpclient.mkdir(basename) self._sftpclient.chdir(basename) return True @Connection.handle_exceptions def rmdir(self, dir_name): """Remove the directory path. Args: dir_name (str): The full directory path. """ pass @Connection.handle_exceptions def reconnect(self): """Reconnect connection.""" if self.connection and not self.connection.get_transport(): self.login() @Connection.handle_exceptions def scan_dir(self, directory, exceptions=[]): """Return list of directories or files. Args: directory (str): The path which you which to scan. exceptions (list): The files or folders to exclude. Returns: list(str) """ results = self._sftpclient.listdir(path=directory) return [x for x in results if x not in exceptions] @Connection.handle_exceptions def transfer_files(self, file_list, transfer_mode='upload', batch_file_id=None, pid=None, overwrite=None, check_delivered_files=True, d_job=None): """Transfer files up or down. Args: file_list (list(dict)): A list of remote/local keypairs transfer_mode (str): upload or download batch_file_id: pid: overwrite: check_delivered_files: d_job: Returns: Raises: Exception: We are unable to transfer files. """ attempts = 1 directories = {os.path.dirname(x['remote']) for x in file_list} for directory in directories: self.mkdir(directory) while attempts <= 5: self.logger.info('Delivery attempt # {}.'.format(attempts)) try: for file in file_list: local_file = file['local'] remote_file = file['remote'] self.logger.info('Start transferring.') if transfer_mode == 'download': self._sftpclient.get(remote_file, local_file) else: if type(local_file) == bytes: self._sftpclient.putfo(io.BytesIO(local_file), remote_file) else: self._sftpclient.put(local_file, remote_file) self.logger.info('Transfer finished.') self.logger.info('Delivery succeeded.') return True except Exception as e: self.logger.error('Error transferring file: {}'.format(e)) attempts += 1 sleep(15) self.logger.info('Delivery failed.') raise exceptions.TransferException()