"""Lambda get-isrc-batches-from-snowflake function module.""" import json import os from datetime import datetime from datetime import timedelta import pandas as pd from lambdacommon.common_config import logger import config from src.connectors.snowflake import SnowflakeClient from src.connectors.s3_client import S3Client queries_path = os.path.dirname(__file__) + '/queries/' def read_query_from_file(file_path): """Read query from file.""" with open(file_path, 'r') as file: return file.read() def get_days_isrcs(day): """Get days isrcs.""" snowflake_client = SnowflakeClient() query = read_query_from_file( queries_path + 'daily_isrc.sql') query = replace_query_placeholder( query, '', day) data_frame = snowflake_client.execute_query(query) snowflake_client.close_connection() isrcs = data_frame[['ISRC', 'UPC']].to_dict('records') isrcs_with_upc = [] last_isrc = '' for isrc in isrcs: if isrc['ISRC'] != last_isrc: isrcs_with_upc.append({ 'isrc': isrc['ISRC'], 'upc': isrc['UPC']}) else: isrcs_with_upc[-1]['upc'] += f', {isrc["UPC"]}' last_isrc = isrc['ISRC'] return isrcs_with_upc def batch_isrcs(isrcs, batch_size): """Batches isrcs.""" batches = [] for i in range(0, len(isrcs), batch_size): batches.append(isrcs[i:i + batch_size]) return batches def replace_query_placeholder(query, placeholder, value): """Replace query placeholder.""" return query.replace(placeholder, value) def get_run_id(snowflake_client): """Get last run id from snowflake table.""" schema_name = config.ENVIRONMENT.upper() query = read_query_from_file( queries_path + 'get_last_run_id.sql') query = replace_query_placeholder( query, '', schema_name) data_frame = snowflake_client.execute_query(query) last_id = data_frame['LAST_RUN_ID'].item() if last_id is None: return 0 else: return last_id + 1 def add_run_data(isrcs, start_time): """Add result columns to isrcs.""" run_id = get_run_id(SnowflakeClient()) time_str = start_time.strftime('%Y-%m-%d %H:%M:%S') for isrc in isrcs: isrc['release_date'] = start_time.strftime('%Y-%m-%d') isrc['ingested_dim_track'] = False isrc['ingested_dim_track_time'] = 'NULL' isrc['ingested_insights'] = False isrc['ingested_insights_time'] = 'NULL' isrc['test_run_id'] = run_id isrc['test_start_time'] = time_str isrc['test_end_time'] = 'NULL' def get_day(event): """Get day from event.""" if 'override_day' in event: return event['override_day'] else: return datetime.now().strftime('%Y%m%d') def get_day_minus(current_day, days_less): """Get day less than input day.""" date_format = '%Y%m%d' date_obj = datetime.strptime(current_day, date_format) date_obj = date_obj - timedelta(days=days_less) return date_obj.strftime(date_format) def upload_to_s3(data, run_id): """Upload data to s3.""" s3_client = S3Client() s3_data = json.dumps(data) s3_key = f'isrcBatchesFromSnowflake-{run_id}.json' s3_client.put_object(s3_key, s3_data) return s3_client.bucket_name, s3_key def get_failed_isrcs_from_previous_days( day, days_less, test_run_id, test_start_time): """Get failed isrcs from previous days.""" oldest_date = get_day_minus(day, days_less) snowflake_client = SnowflakeClient() schema_name = config.ENVIRONMENT.upper() query = read_query_from_file( queries_path + 'get_previous_fails.sql') query = replace_query_placeholder( query, '', schema_name) query = replace_query_placeholder( query, '', oldest_date) isrcs = snowflake_client.execute_query(query) isrcs_to_rerun = build_isrc_array(isrcs.to_dict('records'), test_run_id, test_start_time) snowflake_client.close_connection() return isrcs_to_rerun def get_formatted_time(time): """Get formatted time or return none.""" if time is not None and not pd.isna(time): return time.strftime('%Y-%m-%d %H:%M:%S') return 'NULL' def build_isrc_array(isrcs, test_run_id, test_start_time): """Build isrc array for rerun isrcs.""" isrcs_to_rerun = [] for isrc in isrcs: ingested_dim_track_time = get_formatted_time( isrc['INGESTED_DIM_TRACK_TIME']) ingested_insights_time = get_formatted_time( isrc['INGESTED_INSIGHTS_TIME']) isrcs_to_rerun.append({ 'isrc': isrc['ISRC'], 'upc': isrc['UPC'], 'release_date': isrc['RELEASE_DATE'].strftime('%Y-%m-%d'), 'ingested_dim_track': isrc['INGESTED_DIM_TRACK'], 'ingested_dim_track_time': ingested_dim_track_time, 'ingested_insights': isrc['INGESTED_INSIGHTS'], 'ingested_insights_time': ingested_insights_time, 'test_run_id': test_run_id, 'test_start_time': test_start_time, 'test_end_time': 'NULL', 'rerun': True} ) return isrcs_to_rerun def handler(event, context): """Lambda entry point.""" try: start_time = datetime.now() day = get_day(event) isrcs = get_days_isrcs(day) add_run_data(isrcs, start_time) previous_isrcs = get_failed_isrcs_from_previous_days( day, 7, isrcs[0]['test_run_id'], isrcs[0]['test_start_time']) isrcs = isrcs + previous_isrcs isrc_batches = batch_isrcs(isrcs, int(config.BATCH_SIZE)) run_id = isrcs[0]['test_run_id'] bucket, key = upload_to_s3(isrc_batches, run_id) return {'status': 'OK', 'total_isrcs': len(isrcs), 'num_of_batches': len(isrc_batches), 's3_bucket': bucket, 's3_key': key } except Exception as e: logger.exception(str(e)) raise e