import os import sentry_sdk from config import config from queries.mysql import get_error_jobs_with_queued_claims, update_claim_status_for_job from queries.mysql import update_claims_status from s3.read_s3 import get_file_list, get_file_content log = config.get_logger() sentry_sdk.init(config.SENTRY_DSN) def fix_claims_for_error_job(env): batch_size = 50 bucket_name = f'{env.lower()}-neighbouring-rights' job_count = 0 has_results = True offset = 0 while has_results: log.info(f'Getting jobs that need to be fixed at offset: {offset}') ids, rowcount = get_error_jobs_with_queued_claims( batch_size, offset ) has_results = bool(ids) log.info(f'Found #{rowcount} jobs') if not has_results: break offset += batch_size job_count += rowcount all_files = [] for job in ids: # for each job, get a list of files that are on s3 folder. # it will have error and maybe an excluded files. # all these are old jobs so use the old folder structure. prefix = f"delivery/claims/order_{job['order_id']}_job_{job['job_id']}/" files = get_file_list(bucket_name, prefix) log.info(f"Files in {job['job_id']}: {files}") if len(files) == 0: log.info(f"No files found for job: {job['job_id']}") log.info(f"Going to update all the claims for job: {job['job_id']}") # If there are no files, then update the job status # This function has been giving us a SQL error in the past for jobs with no files. # This should be fixed now but have a look to https://theorchard.atlassian.net/browse/NR-1123 and # https://theorchard.atlassian.net/jira/software/c/projects/NR/boards/488?selectedIssue=NR-1643 # for more information. update_claim_status_for_job(job['job_id'], 'error') else: # if there are files, then update the individual claims statuses all_files += files for file in all_files: rows = get_file_content(bucket_name, file, log) status = 'excluded' if '_excluded.csv' in file else 'error' log.info( f"Updating #{len(rows)} claims from file: {file} with status: {status}") update_claims_status(rows, status) log.info(f'Total {job_count} jobs fixed.') if __name__ == "__main__": """Main entrypoint function.""" env = os.environ.get("ENV", 'qa') try: log.info(f'Starting fix_claims_for_error_job for env={env}') fix_claims_for_error_job(env) except Exception as e: log.error(e) raise e finally: log.info("Finished.")