from multiprocessing import Process

import config
from connectors import mysql, s3

logger = config.logger


def main() -> None:
    database = "accountingflat"
    table = "dig_sales_detail"
    s3_objects = s3.get_s3_objects(
        config.S3_BUCKET, f"{config.S3_OBJECT_PREFIX}/dig_sales_detail/"
    )
    logger.info(f"Objects are {s3_objects}")
    batch_size = config.MYSQL_CONCURRENT_LOADS
    for batch_start in range(0, len(s3_objects), batch_size):
        logger.info(batch_start)
        s3_object_batch = s3_objects[batch_start : batch_start + batch_size]
        logger.info(f"Smaller list is {s3_object_batch}")
        processes = []
        for s3_object in s3_object_batch:
            object_key = f"{config.S3_BUCKET}/{s3_object}"
            sql = f"LOAD DATA FROM S3 FILE 's3://{object_key}' INTO TABLE {database}.{table} FIELDS ESCAPED BY '\\\\' TERMINATED BY '\\t' OPTIONALLY ENCLOSED BY '\"' LINES TERMINATED BY '\\n' (statement_id,date,upc,cd,track_id,isrc,track_name,qty,unit_price,total,trans_type,retail_price,original_price,discount);"
            processes.append(
                Process(target=mysql.load_data, args=(sql, object_key))
            )
        for process in processes:
            process.start()
        for process in processes:
            process.join()


if __name__ == "__main__":
    main()