from multiprocessing import Process

import config
from connectors import mysql, s3

logger = config.logger


def main() -> None:
    database = "accountingflat"
    table = "TEMP_dig_sales_statements"
    s3_objects = s3.get_s3_objects(
        config.S3_BUCKET,
        f"{config.S3_OBJECT_PREFIX}/temp_dig_sales_statements/",
    )
    logger.info(f"Objects are {s3_objects}")
    batch_size = config.MYSQL_CONCURRENT_LOADS
    for batch_start in range(0, len(s3_objects), batch_size):
        logger.info(batch_start)
        s3_object_batch = s3_objects[batch_start : batch_start + batch_size]
        logger.info(f"Smaller list is {s3_object_batch}")
        processes = []
        for s3_object in s3_object_batch:
            object_key = f"{config.S3_BUCKET}/{s3_object}"
            sql = f"LOAD DATA FROM S3 FILE 's3://{object_key}' INTO TABLE {database}.{table} FIELDS ESCAPED BY '\\\\' TERMINATED BY '\\t' OPTIONALLY ENCLOSED BY '\"' LINES TERMINATED BY '\\n' (statement_detail_id, statement_id, DATE, upc, cd, track_id, isrc, track_name, qty, unit_price, total, trans_type, retail_price, original_price, discount, retail, period_id, customer_id, dms_name, vendor_id, owner, owner_id, activity_rate, original_currency_id, release_status);"
            processes.append(
                Process(target=mysql.load_data, args=(sql, object_key))
            )
        for process in processes:
            process.start()
        for process in processes:
            process.join()


if __name__ == "__main__":
    main()