import csv import json import os import sys import uuid import paramiko import pysftp import boto3 from dotenv import load_dotenv load_dotenv() sftp_host = os.getenv('SYNC_HOST') sftp_user = os.getenv('SYNC_USER') sftp_password = os.getenv('SYNC_PASSWORD') def _generate_filename(): unique_name = uuid.uuid4() underscored_name = str(unique_name).replace('-', '_') return underscored_name def fetch_files(input_file): with pysftp.Connection(sftp_host, username=sftp_user, password=sftp_password) as sftp: with open(input_file, newline='') as csvfile: csvreader = csv.DictReader(csvfile, delimiter=',', quotechar='|') for row in csvreader: source_path = row['SOURCE_PATH'] local_path = row['ORIGINAL_FILENAME'] print(f'Fetching {source_path}') sftp.get(source_path, localpath=local_path) print('Done') def upload_files(input_file): with open(input_file, newline='') as csvfile: csvreader = csv.DictReader(csvfile, delimiter=',', quotechar='|') for row in csvreader: local_path = row['ORIGINAL_FILENAME'] metadata = { 'asset_type': 'WAV', 'original_filename': local_path, 'product_id': row['PRODUCT_ID'], 'upc': row['UPC'], 'release_status': 'in_content', 'is_correction': '0', 'track_unique_id': row['TRACK_UNIQUE_ID'] } print(f'Uploading {local_path}') upload_info = get_upload_credentials_and_path() print(f'Got upload_info {upload_info}') s3_client = boto3.client('s3', aws_access_key_id=upload_info['credentials']['aws_access_key_id'], aws_secret_access_key=upload_info['credentials']['aws_secret_access_key'], aws_session_token=upload_info['credentials']['token']) s3_client.upload_file( local_path, upload_info['bucket'], f"{upload_info['filename']}.wav", ExtraArgs={ "Metadata": metadata, "ContentType": 'audio/wav' } ) print('Done') def get_upload_credentials_and_path(): ssh = paramiko.SSHClient() ssh.load_system_host_keys() ssh.connect(os.getenv('SSH_HOST'), username=os.getenv('SSH_USERNAME'), password=os.getenv('SSH_PASSWORD')) ssh_stdin, ssh_stdout, ssh_stderr = ssh.exec_command( "curl -s -d '{ \"asset_type\": \"audio\" }' -H 'Content-Type: application/json' -H 'Orchard-User-Id: oa:1092' -X POST https://prod-ows-assets.theorchard.io/upload-token") result = ssh_stdout.read().decode('ascii') ssh.close() return json.loads(result) def main(input_csv): # print(input_csv) # fetch_files(input_csv) upload_files(input_csv) if __name__ == '__main__': main(sys.argv[1])