from __future__ import annotations import sys import traceback import boto3 from itertools import groupby from typing import TYPE_CHECKING, List from src.config import ENABLED_SERVICE_TYPES, MAX_PLAYLISTS_SYNC_CHUNK_SIZE, SCHEDULE_EXECUTION_SQS_URL from src.enums import ServiceType from src.playlist_sync.services import PlaylistSynchronizationLogService, PlaylistSyncService from src.sqs.sqs_client import SQSClient from src.sqs.utils import parse_events from src.synchronizer import SyncManager from src.utils import func_name if TYPE_CHECKING: from src.logger import BoundLogger from src.playlist_sync.models import PlaylistSynchronization __all__ = ["schedule_all_playlists_sync", "schedule_playlists_sync", "playlists_sync_handler"] def get_syncs(triggered_manually: bool, logger: "BoundLogger") -> list[dict]: service = PlaylistSyncService() def key_func(ps: "PlaylistSynchronization") -> ServiceType | None: if ps.service_account is None: return None return ps.service_account.service_type syncs = [] for service_type, grouped in groupby(service.get_to_sync(), key=key_func): if service_type is None: continue if ENABLED_SERVICE_TYPES and service_type not in ENABLED_SERVICE_TYPES: logger.warning(f"[{func_name()}] Found syncs group of disabled service type {service_type}, skipping.\n" f"Available service types are {ENABLED_SERVICE_TYPES}.") continue grouped = list(grouped) group_size = len(grouped) chunk_size = MAX_PLAYLISTS_SYNC_CHUNK_SIZE or group_size for i in range(0, group_size, chunk_size): syncs.append({ "sync_ids": [ps.id for ps in grouped[i: i + chunk_size]], "triggered_manually": triggered_manually, }) return syncs def schedule_playlists_sync(syncs: List[dict], queue: str, logger: "BoundLogger"): if not syncs: logger.warning(f"[{func_name()}] No syncs to schedule. Stopped.") return logger.info( f"[{func_name()}] Start scheduling synchronization process for syncs:\n {syncs}") sqs_client = SQSClient(provider=boto3.client("sqs"), logger=logger) sqs_client.send(queue, syncs) logger.info(f"[{func_name()}] Successfully sent to SQS {queue}.") def schedule_all_playlists_sync(event: dict, logger: "BoundLogger"): logger.info( f"[{func_name()}] Start scheduling synchronization process for all playlists by event:\n {event}") syncs = get_syncs(triggered_manually=event.get("triggered_manually", False), logger=logger) logger.info(f"[{func_name()}] Got {len(syncs)} syncs to schedule.\nSyncs: {syncs}.") schedule_playlists_sync(syncs, SCHEDULE_EXECUTION_SQS_URL, logger) def run_playlists_sync(sync_ids: List[int], triggered_manually: bool, logger: "BoundLogger"): service = PlaylistSyncService() for sync_id in sync_ids: instance = service.get_with_service_account(sync_id) if instance is None: logger.warning(f"[{func_name()}] Can't find playlist sync with id {sync_id}") return if ( not triggered_manually and ENABLED_SERVICE_TYPES and instance.service_account.service_type not in ENABLED_SERVICE_TYPES ): logger.info(f"[{func_name()}] Skipping sync for {instance.id}, because " f"{instance.service_account.service_type} type is not " f"in available types: {ENABLED_SERVICE_TYPES}.") return log_service = PlaylistSynchronizationLogService() sync_manager = SyncManager(logger, triggered_manually=triggered_manually) logger.info(f"[{func_name()}] Start synchronization process for sync with id {sync_id}") log = sync_manager.synchronize(service, instance, triggered_manually=triggered_manually) logger.info(f"[{func_name()}] Synchronization process for sync with id {sync_id} completed.") log_service.insert_instance(log) logger.info(f"[{func_name()}] Synchronization is finished.") def playlists_sync_handler(event: dict, logger: "BoundLogger"): parsed = parse_events(event, logger) parsed_events_count = len(parsed) failed_events_count = 0 for i, parsed_event in enumerate(parsed): logger.info(f"[{func_name()}][{i}/{parsed_events_count}] Started with parsed event {parsed_event}") try: run_playlists_sync(**parsed_event, logger=logger) except Exception as ex: logger.error(f"[{func_name()}][{i}/{parsed_events_count}] Failed with {ex}.\n" f"{traceback.format_exception(*sys.exc_info()) + traceback.format_stack()}\n" f"Event: {parsed_event}") failed_events_count += 1 continue logger.info(f"[{func_name()}] {parsed_events_count - failed_events_count}/{parsed_events_count} events " f"completed, {failed_events_count} failed.")