""" Script to load Spotify Charts To run: 1. make venv 2. . venv/bin/activate 3. python generate_uow_universal.py --env=dev --version=v2 --config=spotify_config.json --reprocess ( python generate_uow_universal.py -h) The configuration JSON file should have the following structure ( each configuration item configure the single UoW, for each UoW multiple contexts are possible ): [ { "report_name": "charts_daily_regional", "report_date": "2017-02-23", "contexts": [ "global" ] }, { "report_name": "charts_weekly_regional", "report_date": "2017-06-29", "contexts": [ "my", "be" ] } ] """ import argparse import json import logging from datetime import datetime, timedelta, timezone import boto3 from db_schema.schemas import slz from slz_storage.postgres import connection UOW_MOCK = "{dsp}-{day}-{licensor_name}-{report_name}-{version}" CONTEXT = [ "global", "us", "gb", "ae", "ar", "at", "au", "be", "bg", "bo", "br", "by", "ca", "ch", "cl", "co", "cr", "cy", "cz", "de", "dk", "do", "ec", "ee", "eg", "es", "fi", "fr", "gr", "gt", "hk", "hn", "hu", "id", "ie", "il", "in", "is", "it", "jp", "kr", "kz", "lt", "lu", "lv", "ma", "mx", "my", "ng", "ni", "nl", "no", "nz", "pa", "pe", "ph", "pk", "pl", "pt", "py", "ro", "sa", "se", "sg", "sk", "sv", "th", "tr", "tw", "ua", "uy", "ve", "vn", "za" ] # 74 items are in the list DSP = "spotify" LICENSOR = "sme" def run(params: argparse.Namespace): logging.basicConfig(format='[CREATE UOW/CS] %(levelname)s:%(message)s', level=logging.INFO) logging.info("Start script execution") secretsmanager = boto3.client('secretsmanager') with open(params.config) as f: spotify_uow_config = json.load(f) pg = connection.get_session_from_secret_key( secretsmanager, f'delphi/{params.env}/slz/storage/pg_proxy/user') try: for config_item in spotify_uow_config: report_name = config_item["report_name"] report_date = datetime.strptime(config_item["report_date"], '%Y-%m-%d') context_list = config_item["contexts"] licensor_name = LICENSOR now = datetime.now(tz=timezone.utc) report: slz.Report = pg.query(slz.Report).filter( slz.Report.report_name == report_name).one() licensors = { item.licensor_name: item.licensor_id for item in pg.query(slz.Licensor).all() } if not report.data_source.data_source_name == DSP: raise ValueError('dsp and report does not match') already_existed_uow = pg.query(slz.UnitOfWork).filter( slz.UnitOfWork.report_date == report_date.date(), slz.UnitOfWork.report_id == report.report_id, ).with_entities(slz.UnitOfWork.unit_of_work_code).all() already_existed_uow = {res[0] for res in already_existed_uow} # logging.info("Report date: %s", report_date) if report_name == "charts_weekly_regional" and report_date.weekday( ) != 3: logging.info("Skip date: %s", report_date) continue unit_of_work_code = UOW_MOCK.format( dsp=DSP, day=report_date.strftime('%Y%m%d'), licensor_name=licensor_name, report_name=report_name, version=params.version, ) reprocess_id = '' # skip already created if unit_of_work_code in already_existed_uow: if not params.reprocess: logging.info("Already exists, skip: %s", unit_of_work_code) continue logging.info("Already exists, create reprocess UoW: %s", unit_of_work_code) reprocess_id = now.strftime('%Y%m%dT%H%M%S') uow_record = slz.UnitOfWork( unit_of_work_code=unit_of_work_code, reprocess_id=reprocess_id, report_date=report_date.date(), report_id=report.report_id, licensor_id=licensors[licensor_name], version=params.version, activity_status=slz.ActivityStatusEnum.IDLE.value, completeness_status=slz.CompletenessStatusEnum.ACTIVE.value, unit_of_work_type=slz.UnitOfWorkTypeEnum.BACKFILL.value, next_run_at=now + timedelta(minutes=1), created_at=now, last_updated_at=now, is_force_complete=False, priority=slz.UnitOfWorkPriorityEnum.PRIORITY_6.value) for context in CONTEXT: # skip context to load if context in context_list: continue content_status_record = slz.ContentStatus( content_name= f"{context}_{report_date.strftime('%Y%m%d')}.json.gz", context=context, content_status=slz.ContentStatusEnum.ON_HOLD, failure_count=0, latest_job_id='backfill', created_at=now, last_checked_at=now, sub_content=None, metadata_process_status=slz.ContentMetadataStatusEnum. COMPLETED, ) content_status_record.unit_of_work = uow_record logging.info("Add: %s", uow_record.readable) try: pg.add(uow_record) pg.add_all(uow_record.content_statuses) pg.commit() except Exception as err: logging.error("Error during commit: %s", str(err)) pg.rollback() finally: pg.close() logging.info("Finish script execution") if __name__ == '__main__': def date_type(value): try: value = datetime.strptime(value, '%Y%m%d') except Exception: raise argparse.ArgumentTypeError("Not a valid date") return value parser = argparse.ArgumentParser() parser.add_argument("--env", type=str, required=True, choices=['dev', 'qa', 'stage', 'prod']) parser.add_argument("--version", type=str, required=True) parser.add_argument( '--reprocess', action='store_true', help='add this flag to create reprocessing UoW for existing UoW', ) parser.add_argument( "--config", type=str, required=True, help='JSON file with UoW configuration' ) params = parser.parse_args() run(params)