"""Image Asset Migration from S3 Bucket to S3 Bucket. Command line script to identify all images in an S3 bucket an copy to another. Example Usage: $ python image-migration-script.py """ # !/usr/bin/env python # -*- coding: utf-8 -*- import argparse import datetime import hashlib import logging import os from os.path import dirname from os.path import join import re import sys import time import uuid import boto3 from dotenv import load_dotenv import requests dotenv_path = join(dirname(__file__), '.env') load_dotenv(dotenv_path) OWS_PRODUCT_URL = os.environ.get('OWS_PRODUCT_URL') SOURCE_BUCKET_NAME = os.environ.get('SOURCE_BUCKET_NAME') SOURCE_SUB_FOLDER = os.environ.get('SOURCE_SUB_FOLDER') DESTINATION_BUCKET_NAME = os.environ.get('DESTINATION_BUCKET_NAME') DESTINATION_SUB_FOLDER = os.environ.get('DESTINATION_SUB_FOLDER') def fetch_product_id_by_upc(upc, correlation_id): """Return a product id by making a request to ows-product. Args: upc (str): upc for the product to be retrieved correlation_id (uuid): unique identifier for requests made Returns: product_id (str) or None: the product_id of the upc or None if error """ request_url = '{}/product/upc/{}'.format(OWS_PRODUCT_URL, upc) headers = { 'Content-Type': 'application/json', 'Correlation-Id': correlation_id} product_response = requests.get( request_url, headers=headers) if product_response.status_code != 200: logging.warning('failed to find upc {}, status was {}.\n'.format( upc, product_response.status_code)) return None return str(product_response.json().get('product_id')) def save_asset_to_destination_bucket( s3_object, filename, product_id, file_extension, src_bucket_name, destination_bucket_name, destination_sub_folder): """Save an asset to the specified destination bucket. Args: s3_object (class): AWS S3 object to be copied to new destination filename (str): filename of asset object to be copied product_id (str): product_id of asset to be copied file_extension (str): file extension of original file src_bucket_name (str): name of the source bucket destination_bucket_name (str): name of the destination bucket destination_sub_folder (str): folder where copied image will be placed """ image_type = re.search('(?<=\/)(\w+)', s3_object.key).group(0) md5_product_id = hashlib.md5(product_id.encode('utf-8')).hexdigest() destination_filename = '{}{}'.format(md5_product_id, file_extension) copy_source = { 'Bucket': src_bucket_name, 'Key': s3_object.key } s3.meta.client.copy( copy_source, destination_bucket_name, '{destination_sub_folder}/{image_type}/{filename}'.format( destination_sub_folder=destination_sub_folder, image_type=image_type, filename=destination_filename)) def step_through_images_from_s3_bucket_and_save( source_bucket, source_bucket_name, source_sub_folder, destination_bucket_name, destination_sub_folder, max_objects): """Loop through objects in an AWS S3 bucket and save to another bucket. Args: source_bucket (class): AWS S3 Bucket to copy objects from source_bucket_name (str): name of source bucket source_sub_folder (str): name of folder to search within destination_bucket_name (str): name of destination bucket destination_sub_folder (str): name of folder to save objects to max_objects (int): number of objects to copy """ start_time = time.time() count = 0 correlation_id = str(uuid.uuid4()) logging.warning('Date of execution: {}'.format(datetime.datetime.now())) logging.warning('Now running migration with Correlation-Id: {}'.format( correlation_id)) for obj in source_bucket.objects.filter(Prefix=source_sub_folder): # Look for S3 objects that end with .jpg or .jpeg file_match = re.search('[^\/]*\..+$', obj.key, re.IGNORECASE) filename = file_match.group(0) file_extension = re.search('\..+$', filename).group(0) if file_match: upc = re.search('\d+', filename).group(0) # Fetch the product_id based on the upc from ows-product product_id = fetch_product_id_by_upc(upc, correlation_id) # If no product is found, continue looping through the bucket if product_id is None: continue save_asset_to_destination_bucket( obj, filename, product_id, file_extension, source_bucket_name, destination_bucket_name, destination_sub_folder) count += 1 if count == max_objects: break run_time = time.time() - start_time logging.warning('copied {} objects in {} seconds'.format(count, run_time)) sys.exit(0) if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument( 'max_objects', nargs='?', default=0, type=int, help='the max number of objects copied') args = parser.parse_args() s3 = boto3.resource('s3') source_bucket = s3.Bucket(SOURCE_BUCKET_NAME) logging.basicConfig( filename='ows-product-calls.log', level=logging.WARNING) step_through_images_from_s3_bucket_and_save( source_bucket, SOURCE_BUCKET_NAME, SOURCE_SUB_FOLDER, DESTINATION_BUCKET_NAME, DESTINATION_SUB_FOLDER, args.max_objects)