import boto3
from boto3.s3.transfer import TransferConfig
from smart_open import open

client = boto3.client('s3')
source = './spotify_2019-08-22_sony_v2.2_us_streams.ndjson'
# dest = 's3://dev-sme-data-archive/spotify/streams/v2/report_date=2018-08-29/report_licensor=smejp/streams_20180829_JP.gz'
dest = 's3://dev-sme-data-archive/spotify/streams_20180829_JP.gz'
bucket, *tail = dest.strip("s3://").split('/', 1)
key = tail[0]
# chunk_size = 8388608 # 8*1024*1024
chunk_size = 80 * 1024 * 1024

# config = TransferConfig(max_concurrency=10, use_threads=True)
# response = client.upload_file(source, bucket, key, Config=config)
with open(source, 'rb') as f:
    with open(dest, 'wb') as fout:
        while True:
            data = f.read(chunk_size)
            if not len(data):
                break
            fout.write(data)