"""Make backup archive and upload to S3.""" import datetime import glob import logging import os import tarfile import time from logging.handlers import SMTPHandler import boto3 import botocore import backup_config as config backup_source = config.BACKUP_SOURCE_DIRECTORY def check_and_remove_backup(source_file): """ Sanity-check file against S3 object and remove local file. Args: source_file: backup file to check and remove """ s3 = boto3.client('s3', config=botocore.client.Config( signature_version='s3v4')) bucket = config.S3_BUCKET s3_key = ''.join([config.S3_KEY, '/', source_file.rsplit('/', 1)[-1]]) try: status = s3.head_object(Bucket=bucket, Key=s3_key) local_file_size = os.path.getsize(source_file) remote_file_size = status.get('ContentLength') # If file sizes are the same, remove local file if local_file_size == remote_file_size: os.remove(source_file) logging.info('Removed local file: {}'.format(source_file)) except botocore.exceptions.ClientError as error: logging.error('Retrieval from S3 failed: {}'.format(error)) return None def make_backup_tar(source_directory): """ Make tar archive of backup directory. Args: source_directory: backup directory path """ try: now = time.time() file_name = ''.join(['/tmp/', datetime.datetime.now().strftime( "%Y_%m_%d_%H-%M-%S"), '.tar.gz']) backup_archive = tarfile.open(file_name, 'w:gz') for file in glob.glob(os.path.join(source_directory, '*.tar.gz')): # Only add files from the past 24 hours file_mtime = os.path.getmtime(file) if now - file_mtime < 86400: backup_archive.add(file, arcname=os.path.basename(file)) except Exception as error: logging.error('Could not create archive: {}'.format(error)) return file_name def upload_to_s3(file_to_upload): """Upload backup file to S3. Args: file_to_upload (str): path to file for upload Return: object: the resulting S3 object """ s3 = boto3.client('s3', config=botocore.client.Config( signature_version='s3v4')) bucket = config.S3_BUCKET s3_key = ''.join([config.S3_KEY, '/', file_to_upload.rsplit('/', 1)[-1]]) kms_key_id = config.KMS_KEY_ID try: file = open(file_to_upload, 'rb') new_object = s3.put_object( ACL='private', Bucket=bucket, Key=s3_key, ServerSideEncryption='aws:kms', SSEKMSKeyId=kms_key_id, Body=file) except botocore.exceptions.ClientError as error: logging.error('Upload to S3 failed: {}'.format(error)) return new_object mail_handler = SMTPHandler( mailhost=config.MAIL_HOST, fromaddr=config.FROM_ADDR, toaddrs=config.TO_ADDRS, subject='Infoblox Backup Error' ) logging.basicConfig(filename=config.LOGFILE_NAME, level=logging.INFO) mail_handler.setLevel(logging.ERROR) mail_handler.setFormatter(logging.Formatter( '[%(asctime)s] %(levelname)s: %(message)s' )) logger = logging.getLogger() logger.addHandler(mail_handler) try: backup_file = make_backup_tar(backup_source) file = upload_to_s3(backup_file) logging.info('Uploaded S3 object: {}'.format(file)) check_and_remove_backup(str(backup_file)) except Exception as error: logger.critical('Backup routine failed: {}'.format(error))