"""Step 2 of Image Asset Migration from S3 Bucket to S3 Bucket. Command line script to create paths for images in an S3 bucket and write batch files for further processing. Example Usage: $ python create_batch_paths_script.py """ # !/usr/bin/env python # -*- coding: utf-8 -*- import argparse import datetime import hashlib import logging import os from os.path import dirname from os.path import join import re import sys import time import uuid import boto3 from dotenv import load_dotenv import requests dotenv_path = join(dirname(__file__), '.env') load_dotenv(dotenv_path) OWS_PRODUCT_URL = os.environ.get('OWS_PRODUCT_URL') SOURCE_BUCKET_NAME = os.environ.get('SOURCE_BUCKET_NAME') SOURCE_SUB_FOLDER = os.environ.get('SOURCE_SUB_FOLDER') DESTINATION_BUCKET_NAME = os.environ.get('DESTINATION_BUCKET_NAME') DESTINATION_SUB_FOLDER = os.environ.get('DESTINATION_SUB_FOLDER') def fetch_product_id_by_upc(upc, correlation_id): """Return a product id by making a request to ows-product. Args: upc (str): upc for the product to be retrieved correlation_id (uuid): unique identifier for requests made Returns: product_id (str) or None: the product_id of the upc or None if error """ request_url = '{}/product/upc/{}'.format(OWS_PRODUCT_URL, upc) headers = { 'Content-Type': 'application/json', 'Correlation-Id': correlation_id} product_response = requests.get( request_url, headers=headers) if product_response.status_code != 200: logging.warning('failed to find upc {}, status was {}.\n'.format( upc, product_response.status_code)) return None return str(product_response.json().get('product_id')) def step_through_images_from_s3_bucket_and_create_batch_files( file_name, destination_bucket_name, destination_sub_folder): """Loop through objects in an AWS S3 bucket and create batch files for further processing. Args: file_name (str): key file to be processed destination_bucket_name (str): name of destination bucket destination_sub_folder (str): name of folder to save objects to """ start_time = time.time() file_prefix = 'image-asset-migration-batch-paths-' logging.warning('Date of execution: {}'.format(datetime.datetime.now())) with open(file_name) as f: lines = f.readlines() f_new = open(file_prefix + file_name.split('batch-key-', 1)[1], 'w') for line in lines: bucket, key, upc, file_extension, correlation_id = line.strip().split(',') logging.warning('Now running migration with Correlation-Id: {}'.format( correlation_id)) # Fetch the product_id based on the upc from ows-product product_id = fetch_product_id_by_upc(upc, correlation_id) # If no product is found, continue looping through the bucket if product_id is None: continue image_type = re.search('(?<=\/)(\w+)', key).group(0) md5_product_id = hashlib.md5(product_id.encode('utf-8')).hexdigest() destination_filename = '{}{}'.format(md5_product_id, file_extension) destination_file_path = '{destination_sub_folder}/{image_type}/{filename}'.format( destination_sub_folder=destination_sub_folder, image_type=image_type, filename=destination_filename) file_line = '{bucket},{key},{destination_bucket_name},{destination_file_path},{correlation_id}'.format( bucket=bucket, key=key, destination_bucket_name=destination_bucket_name, destination_file_path=destination_file_path, correlation_id=correlation_id) f_new.write(file_line + os.linesep) f_new.close() run_time = time.time() - start_time logging.warning('wrote file in {} seconds'.format(run_time)) sys.exit(0) if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument( 'file_name', nargs='?', default=1000, type=str, help='the key file to be processed') args = parser.parse_args() logging.basicConfig( filename='create_batch_paths.log', level=logging.WARNING) step_through_images_from_s3_bucket_and_create_batch_files( args.file_name, DESTINATION_BUCKET_NAME, DESTINATION_SUB_FOLDER)