import csv from concurrent import futures from functools import wraps import json import os import sys import threading import boto3 import botocore import requests MEZZANINE_BUCKET_NAME = 'qa-orcd-mezzanine-assets' s3_client = boto3.client('s3') _DEFAULT_POOL = futures.ThreadPoolExecutor(max_workers=10) def threadpool(f, executor=None): @wraps(f) def wrap(*args, **kwargs): return (executor or _DEFAULT_POOL).submit(f, *args, **kwargs) return wrap class ProgressPercentage(object): def __init__(self, filename): self._filename = filename self._size = float(os.path.getsize(filename)) self._seen_so_far = 0 self._lock = threading.Lock() def __call__(self, bytes_amount): # To simplify we'll assume this is hooked up # to a single filename. with self._lock: self._seen_so_far += bytes_amount percentage = (self._seen_so_far / self._size) * 100 print( "\r%s %s / %s (%.2f%%)" % ( self._filename, self._seen_so_far, self._size, percentage)) #sys.stdout.flush() def get_assets_host(env): return 'https://{}-ows-assets.theorchard.io'.format(env) def get_full_filename(token_response_body, upload_file): return '{}.{}'.format( token_response_body.get('filename'), upload_file.get('asset_type') ) def create_token(): request_headers = { 'Grass-Account-Type': 'vendor', 'Grass-Account-Id': '25824', 'Content-Type': 'application/json', 'Correlation-Id': 'abc123', 'Orchard-User-Id': 'alw:43807' } token_response = requests.post( '{}/upload-token'.format(get_assets_host('qa')), headers=request_headers, data=json.dumps({ 'asset_type': 'image' }) ) return token_response.json() def upload(product, upload_file, token_response_body): # upload_creds = token_response_body.get('credentials') bucket_name = token_response_body.get('bucket') dest_file_name = get_full_filename(token_response_body, upload_file) file_to_upload = upload_file.get('file') s3_metadata = { 'asset_type': upload_file.get('asset_type').upper(), 'product_id': product.get('product_id'), 'original_filename': upload_file.get('original_filename'), 'upc': product.get('upc'), 'track_unique_id': product.get('tuid') } print('dest file: {}'.format(dest_file_name)) print('s3 metadata: {}'.format(json.dumps(s3_metadata))) print('content type: {}'.format(upload_file.get('content_type'))) s3_client.upload_file( file_to_upload, bucket_name, dest_file_name, Callback=ProgressPercentage(file_to_upload), ExtraArgs={ 'Metadata': s3_metadata, 'ContentType': upload_file['content_type'] } ) def download_mezzanine(mezz_file): file = 'qa/{}'.format(mezz_file['original_filename']) try: s3_client.download_file( MEZZANINE_BUCKET_NAME, mezz_file['filename'], file ) return file except botocore.exceptions.ClientError as e: if e.response['Error']['Code'] == "404": print("The object does not exist.") return False def process_file(test_file): downloaded_file = download_mezzanine(test_file) if downloaded_file: token_response_body = create_token() product = { 'product_id': test_file['product_id'], 'upc': test_file['upc'], 'tuid': '0' } upload_file = { 'file': downloaded_file, 'original_filename': test_file['original_filename'], 'asset_type': test_file['asset_type'], 'content_type': test_file['content_type'] } upload(product, upload_file, token_response_body) return True if __name__ == "__main__": input_file = sys.argv[1] column_names = [] files = [] with open(input_file) as csv_file: csv_reader = csv.reader(csv_file, delimiter=',') line_count = 0 for row in csv_reader: if line_count == 0: column_names = row line_count += 1 else: file = {} for idx, key in enumerate(column_names): file[key] = row[idx] file['original_filename'] = '{}.tif'.format(file['original_filename'].split('.')[0]) files.append(file) line_count += 1 # print(files) print(f'Processed {line_count} lines.') for file in files: process_file(file) sys.exit(0)