import boto3
from boto3.s3.transfer import TransferConfig
from smart_open import open

client = boto3.client('s3')
source = './spotify_2019-08-22_sony_v2.2_us_streams.ndjson'
# dest = 's3://dev-sme-data-archive/spotify/streams/v2/report_date=2018-08-29/report_licensor=smejp/streams_20180829_JP.gz'
dest = 's3://dev-sme-data-archive/spotify/manual.gz'
bucket, *tail = dest.strip("s3://").split('/', 1)
key = tail[0]
# chunk_size = 8388608 # 8*1024*1024
chunk_size = 80 * 1024 * 1024

# config = TransferConfig(max_concurrency=10, use_threads=True)
# response = client.upload_file(source, bucket, key, Config=config)
with open(source, 'rb') as f:
    mpu = client.create_multipart_upload(Bucket=bucket, Key=key)
    mpu_id = mpu['UploadId']
    i = 1
    parts = []
    total = 0
    try:
        while True:
            data = f.read(chunk_size)
            if not len(data):
                break
            part = client.upload_part(
                Body=data, Bucket=bucket, Key=key, UploadId=mpu_id, PartNumber=i
            )
            parts.append({"PartNumber": i, "ETag": part["ETag"]})
            total += len(data)
            # print('uploaded: ', total)
            i += 1
    finally:
        client.complete_multipart_upload(
            Bucket=bucket, Key=key, UploadId=mpu_id, MultipartUpload={"Parts": parts}
        )
        print("finished:", total)