#!/usr/bin/python3

import os,sys,re
import json
import boto3
import logging

# include path to mysql connect
sys.path.append('/app/tools/python/lib')

from rsdb.connect.db import getConnection, getExecCursor

BUCKET = 'orchard-transfers'
SUBDIR = 'catalog-deliveries'
LOCAL = '/app/shared/ddex_import'

logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s', datefmt='[%F %I:%M:%S]', level=logging.INFO)

'''
Get a bucket object to use for list,get,create
'''
session = boto3.session.Session(profile_name='orch-transfer')
s3      = session.resource('s3')
bucket  = s3.Bucket(BUCKET)

sql = "SELECT * FROM RSCOMMON.orchard_client WHERE ddex_source IS NOT NULL ORDER BY ddex_source"
cxn = getConnection('dbmaster')
cur = getExecCursor(cxn, sql)

for row in cur:

    ddexDir = SUBDIR + '/' + row.ddex_source

    logging.info("Reading S3 directory {}".format(ddexDir))

    for awsFile in bucket.objects.filter(Prefix=ddexDir):

        # client directory under catalog-deliveries
        s3ParentDir = os.path.dirname(awsFile.key)

        # only grab the files with parentDir == ddexDir - find --maxdepth 1
        if s3ParentDir != ddexDir:
            continue

        # just grab the XML files
        if not re.match(r".*\.xml$", awsFile.key):
            continue

        # local path
        filename = os.path.basename(awsFile.key)
        localpath = os.path.join(LOCAL, row.ddex_dest, filename)
        if os.path.isfile(localpath):
            stat = os.stat(localpath)
            if stat.st_size != awsFile.size:
                logging.error(
                    "{} size {} is different than {} AWS file".format(localpath, stat.st_size, awsFile.size))
                logging.warn(" o removing {} and downloading again".format(filename))
                os.remove(localpath)
            else:
                continue

        # copy from S3 to localpath
        logging.info(" o downloading " + localpath)
        bucket.download_file(awsFile.key, localpath)

        # move file to date folder - create folder if necessary
        dirname = os.path.dirname(awsFile.key)
        awsFileNewKey = dirname + '/' + awsFile.last_modified.strftime("%Y%m%d") + '/' + filename
        logging.info(" o moving {}".format(awsFileNewKey))
        s3.meta.client.copy(CopySource={'Bucket': BUCKET, 'Key': awsFile.key}, Bucket=BUCKET, Key=awsFileNewKey)

        # if the file was successfully copied - delete the source file
        try:
            client = session.client('s3')
            client.head_object(Bucket=BUCKET, Key=awsFileNewKey)
            awsFile.delete()
        except ClientError as err:
            if exc.response['Error']['Code'] != '404':
                raise


cur.close