"""Lambda GitHub repositories backup function module.""" import logging import os import re import time from os.path import exists import boto3 from github3 import login from requests.exceptions import ConnectionError from requests.exceptions import ReadTimeout import sentry_sdk import config sentry_sdk.init( dsn=config.SENTRY_DSN, traces_sample_rate=1.0, ) # Initialize s3 client s3 = boto3.client('s3') def backup_repo(repo, repo_name, bucket_name): """Archive a single GitHub repo and upload it to S3, retrying on transient network errors.""" for attempt in range(1, config.ARCHIVE_MAX_RETRIES + 1): try: repo.archive('zipball', path=repo_name, ref='master') if exists(repo_name): s3.upload_file(repo_name, bucket_name, f'{repo_name}/{repo_name}', ExtraArgs={'ServerSideEncryption': 'aws:kms', 'SSEKMSKeyId': config.KMS_KEY_ID}) logging.info(f'The {repo_name} repository was archived and uploaded to S3') os.remove(repo_name) else: logging.info(f'The {repo_name} repository is empty') return except (ReadTimeout, ConnectionError) as e: if exists(repo_name): os.remove(repo_name) if attempt == config.ARCHIVE_MAX_RETRIES: raise wait = config.ARCHIVE_RETRY_WAIT * (2 ** (attempt - 1)) logging.warning( f'Attempt {attempt}/{config.ARCHIVE_MAX_RETRIES} failed for {repo_name} ' f'({type(e).__name__}), retrying in {wait}s' ) time.sleep(wait) def main(): """Replicate all GitHub repos to s3 bucket.""" access_token = config.GITHUB_TOKEN github = login(token=access_token) github.session.default_read_timeout = config.ARCHIVE_READ_TIMEOUT org = github.organization(config.GITHUB_ORGANIZATION) bucket_name = config.S3_BUCKET_NAME logging.basicConfig( level=logging.INFO, format='%(asctime)s%(levelname)s%(message)s', datefmt='%Y-%m-d% %H:%M:%S' ) # Download and upload each repository to S3 for i in org.repositories(type='all'): repo_name = re.sub('^theorchard/', '', str(i)) backup_repo(i, repo_name, bucket_name) if __name__ == '__main__': main()