"""script used for generating text files that were once used by a single fargate task that would use
multiprocessing to process N number of files listed in a given text file.
"""
import os

import config

PAGE_SIZE = 100
LIMIT = 5000
CHUNK_SIZE = 25
BUCKET = 'dev-orcd-raw-assets'
PREFIX = '2020-07-20//'

paginator_config = {
    'PageSize': PAGE_SIZE
}


def chunks(lst, n):
    """Yield successive n-sized chunks from lst."""
    for i in range(0, len(lst), n):
        yield lst[i:i + n]


if __name__ == '__main__':
    paginator = config.client.get_paginator('list_objects')
    page_iterator = paginator.paginate(
        Bucket=BUCKET,
        Prefix=PREFIX,
        PaginationConfig=paginator_config
    )
    files = []
    for page in page_iterator:
        items = page['Contents']  # One page of image files.
        for item in items:
            key = item['Key']
            response = config.client.head_object(
                Bucket=BUCKET,
                Key=key
            )
            metadata = response['Metadata']
            if 'fingerprinted' not in metadata or metadata['fingerprinted'] != '1':
                files.append(key)
                print(key)

            if len(files) == LIMIT:
                break

        if len(files) == LIMIT:
            break
    chunked_files = list(chunks(files, CHUNK_SIZE))
    for i in range(len(chunked_files)):
        chunk = chunked_files[i]
        directory = 'batches/chunk-{}/'.format(i)
        os.mkdir(directory)
        with open('{}/batch.txt'.format(directory), 'w') as txt_file:
            for file in chunk:
                txt_file.write(file + '\n')