"""Lambda generate_hfa_tmp_files function module.""" import json import datetime from typing import Any, Dict, List, Tuple import pymysql from sentry_sdk import capture_exception import config from lambdacommon.common_config import logger from lambdacommon.aws import s3 from lambdacommon import util from src.connector.graphql import execute_graphql_query from src.queries.cypher import SYNCED_TRACKS from src.queries.graphql_queries import OBTAIN_PENDING_TRACK_ID_DATA from src.queries.sql_queries import OBTAIN_HFA_PENDING_RESUBMIT_REQUEST from src.base import run_cypher_query from src.constants import ( FILES_TO_CLEANUP, GENERATE_HFA_TMP_FILES, GENERATE_HFA_TMP_FILES_LENGTH, HFA_ORCHARD_TRACK_LICENSES_FILE, OK, PENDING_REQ_TRACK_IDS_DATA_TEMPLATE, PENDING_REQUEST_TRACK_IDS_DATA_FILE, STATUS, TMP_FILE_TEMPLATE, ) util.init_sentry_for_lambda() def neo4j_synced_tracks(track_ids: List[str]) -> List[str]: """Check synced tracks in Neo4j from art_relation. Args: track_ids (List[str]): List of track IDs (TUIDs). Returns: List[str]: Synced track IDs found in Neo4j. """ try: result = run_cypher_query( SYNCED_TRACKS, {'track_ids': track_ids} ) if not result: logger.warning('No results returned from Neo4j for TUIDs: %s', track_ids) return [] if 'collect(t.id)' not in result[0]: logger.error("Expected key 'collect(t.id)' not found in Neo4j result: %s", result) return [] synced_ids = result[0]['collect(t.id)'] logger.info('Found %d synced tracks in Neo4j out of %d input TUIDs.', len(synced_ids), len(track_ids)) return synced_ids except Exception: logger.exception('Error querying Neo4j for synced tracks') raise def query_graphql_for_track_data(track_ids: List[str]) -> List[Dict[str, Any]]: """Query graphql-gateway to get track data for the given track IDs. Args: track_ids (List[str]): List of track IDs to query. Returns: List[Dict[str, Any]]: Track data results from GraphQL. """ try: logger.info('Sending GraphQL request with %d track_ids', len(track_ids)) variables = {'track_ids': track_ids} data = execute_graphql_query(query=OBTAIN_PENDING_TRACK_ID_DATA, variables=variables) tracks = data.get('tracks', []) logger.info('GraphQL query returned %d tracks', len(tracks)) return tracks except Exception: logger.exception('Unexpected error in query_graphql_for_track_data.') raise def get_pending_hfa_request_from_publishing() -> List[Tuple[Any, ...]]: """Query art_relations to get pending HFA resubmit requests. Returns: List[Tuple]: Rows from the SQL query. """ try: db_credentials = config.PB_DB_CREDENTIALS required_keys = ['host', 'user', 'password', 'database'] if not all(k in db_credentials for k in required_keys): logger.error('Incomplete DB credentials in config.PB_DB_CREDENTIALS') raise ValueError('Missing required database credentials') logger.debug('Connecting to publishing database at host: %s', db_credentials['host']) with util.mysql_connection( db_credentials['host'], db_credentials['user'], db_credentials['password'], db_credentials['database'] ) as conn: with conn.cursor() as cursor: logger.info('Executing HFA pending resubmit request query.') cursor.execute( OBTAIN_HFA_PENDING_RESUBMIT_REQUEST, (config.DATE_FILTER,) ) results = cursor.fetchall() return results except pymysql.MySQLError: logger.exception('Database error while fetching HFA requests') raise except Exception: logger.exception('Unexpected error in get_pending_hfa_request_from_publishing') raise def filter_synced_results( all_results: List[Dict[str, Any]], synced_ids: List[str] ) -> List[Dict[str, Any]]: """ Filter the list of all track results to include only those that are synced. Args: all_results (List[Dict[str, Any]]): List of all track result dictionaries. synced_ids (List[str]): List of synced orchard track IDs. Returns: List[Dict[str, Any]]: Filtered list containing only synced track results. """ return [row for row in all_results if row['orchard_track_id'] in synced_ids] def upload_to_s3(key: str, data: List[Dict[str, Any]]) -> str: """ Upload data to an S3 bucket under a specific key. Args: key (str): The S3 object key (filename). data (List[Dict[str, Any]]): The data to upload, serialized as JSON. Returns: str: The S3 object key (e.g., tmp_dir/filename.json) """ s3_key = f'{config.S3_TMP_FILE_DIR}{key}' s3_full_path = f's3://{config.S3_BUCKET_NAME}/{s3_key}' logger.info(f'Uploading 1 item to S3: {s3_full_path} with dictionary length: {len(data)}') s3.put_object( bucket=config.S3_BUCKET_NAME, key=s3_key, data=json.dumps(data, ensure_ascii=False) ) return s3_key def fetch_synced_track_data_in_batches(synced_ids: List[str]) -> List[Dict[str, Any]]: """ Fetch detailed track data for a list of synced track IDs via GraphQL in batches. Args: synced_ids (List[str]): List of synced orchard track IDs. Returns: List[Dict[str, Any]]: Combined track data returned from GraphQL. """ track_data = [] for i in range(0, len(synced_ids), config.BATCH_SIZE): batch = synced_ids[i:i + config.BATCH_SIZE] try: logger.info(f'Fetching GraphQL data for batch {i // config.BATCH_SIZE + 1}') data = query_graphql_for_track_data(batch) track_data.extend(data) except Exception as e: logger.error(f'GraphQL batch fetch failed for batch starting at index {i}: {e}') raise e return track_data def handler(event, context): """Lambda entry point.""" try: file_date = datetime.datetime.now().strftime('%Y%m%d%H%M%S') tmp_file = TMP_FILE_TEMPLATE.format(date=file_date) pending_req_file = PENDING_REQ_TRACK_IDS_DATA_TEMPLATE.format(date=file_date) generated_temp_files = {} files_to_cleanup = [] result = get_pending_hfa_request_from_publishing() if not result: logger.info('No pending HFA requests found.') return { STATUS: OK, GENERATE_HFA_TMP_FILES: generated_temp_files, GENERATE_HFA_TMP_FILES_LENGTH: len(generated_temp_files), FILES_TO_CLEANUP: files_to_cleanup } logger.info(f'Found {len(result)} pending HFA requests.') track_ids = [row['orchard_track_id'] for row in result] synced_tracks = neo4j_synced_tracks(track_ids) unsynced_count = len(track_ids) - len(synced_tracks) logger.info(f'{unsynced_count} tracks are not synced in Neo4j ({config.ENVIRONMENT}).') synced_track_data = filter_synced_results(result, synced_tracks) tmp_s3_key_path = upload_to_s3(tmp_file, synced_track_data) generated_temp_files[HFA_ORCHARD_TRACK_LICENSES_FILE] = tmp_file files_to_cleanup.append(tmp_s3_key_path) logger.info(f'Generated temporary file: {tmp_file} and uploaded to S3') track_data = fetch_synced_track_data_in_batches(synced_tracks) pending_s3_key_path = upload_to_s3(pending_req_file, track_data) generated_temp_files[PENDING_REQUEST_TRACK_IDS_DATA_FILE] = pending_req_file files_to_cleanup.append(pending_s3_key_path) logger.info(f'Generated temporary file: {pending_req_file} and uploaded to S3') return { STATUS: OK, GENERATE_HFA_TMP_FILES: generated_temp_files, GENERATE_HFA_TMP_FILES_LENGTH: len(generated_temp_files), FILES_TO_CLEANUP: files_to_cleanup } except Exception as e: capture_exception(e) logger.exception(str(e)) raise