"""Celery task definitions for playlist synchronization. Both tasks mirror the behavior of ``PlaylistSynchronizationTask.cs`` and ``PlaylistSynchronizationManager.ExecutePlaylistSyncAsync``. """ from __future__ import annotations import logging import os from datetime import datetime, timezone from typing import TYPE_CHECKING, Optional if TYPE_CHECKING: from playlist_sync.models.generic_playlist import GenericPlaylist from playlist_sync.models.sync_task import PlaylistSynchronization import sentry_sdk from celery import Celery from sentry_sdk.integrations.celery import CeleryIntegration from playlist_sync.config import settings logger = logging.getLogger("playlist_sync_worker") celery_app = Celery( "playlist_sync_worker", broker=settings.CELERY_BROKER_URL, backend=settings.CELERY_RESULT_BACKEND, ) celery_app.conf.update( task_serializer="json", accept_content=["json"], result_serializer="json", timezone="UTC", enable_utc=True, task_track_started=True, beat_schedule_filename="/tmp/celerybeat-schedule", ) if settings.SYNC_SWEEP_INTERVAL_SECONDS > 0: from datetime import timedelta celery_app.conf.beat_schedule = { "periodic-sync-sweep": { "task": "sync_task.periodic_sweep", "schedule": timedelta(seconds=settings.SYNC_SWEEP_INTERVAL_SECONDS), }, } if settings.SENTRY_DSN: sentry_sdk.init( dsn=settings.SENTRY_DSN, environment=settings.SENTRY_ENVIRONMENT, traces_sample_rate=0.0, integrations=[ CeleryIntegration(), ], ) @celery_app.task(name="sync_task.execute_single", bind=True, max_retries=3) def execute_single_sync_task(self, sync_id: int, triggered_manually: bool = False): """Execute a single playlist synchronization. Mirrors: PlaylistSynchronizationManager.ExecutePlaylistSyncAsync() Steps: 1. Load PlaylistSynchronization by sync_id; skip if deactivated mid-run. 2. Load the target ServiceAccount. 3. Validate account.application_id == sync.application_id. 4. Fetch source Spotify playlist via Spotify Web API → GenericPlaylist. 5. Route to the correct synchronizer based on account.music_service_id: - 1 (Spotify) → SpotifySynchronizer - 2 (Deezer) → DeezerSynchronizer - 3 (YouTube) → YoutubeSynchronizer - 4 (SoundCloud) → SoundCloudSynchronizer 6. Update sync row: SourceTitle, SourceImage, SourceTrackCount, SynchronizedTrackCount, LastUpdated, Error. 7. Write a log row to tblPlaylistSynchronizationLog. """ import asyncio from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine from sqlalchemy.orm import sessionmaker as async_sessionmaker from sqlalchemy.pool import NullPool from playlist_sync.models.enums import MusicService, SyncError from playlist_sync.models.exceptions import PlaylistSynchronizationException from playlist_sync.models.playlist_sync_result import PlaylistSynchronizationResult from playlist_sync.models.service_account import ServiceAccount from playlist_sync.models.sync_log import PlaylistSynchronizationLog from playlist_sync.models.sync_task import PlaylistSynchronization from playlist_sync.synchronizers.deezer import DeezerSynchronizer from playlist_sync.synchronizers.soundcloud import SoundCloudSynchronizer from playlist_sync.synchronizers.spotify import SpotifySynchronizer from playlist_sync.synchronizers.youtube import YoutubeSynchronizer async def _run(): async_engine = create_async_engine(settings.DATABASE_URL, poolclass=NullPool) try: async_session_factory = async_sessionmaker( async_engine, class_=AsyncSession, expire_on_commit=False ) async with async_session_factory() as session: # 1. Load sync from sqlmodel import select result = await session.execute( select(PlaylistSynchronization).where( PlaylistSynchronization.id == sync_id ) ) sync = result.scalar_one_or_none() if not sync: logger.warning("Sync %d not found, skipping.", sync_id) return {"status": "skipped", "reason": "not_found"} if not sync.active: logger.debug("Sync %d is inactive, skipping.", sync_id) return {"status": "skipped", "reason": "inactive"} logger.info( "Executing sync %d: %s → %s", sync_id, sync.from_playlist_id, sync.to_playlist_id, ) # 2. Load service account result = await session.execute( select(ServiceAccount).where( ServiceAccount.id == sync.to_service_account_id ) ) account = result.scalar_one_or_none() sync_result: PlaylistSynchronizationResult try: if not account: raise PlaylistSynchronizationException( f"Could not find service account" f" {sync.to_service_account_id} for sync {sync_id}", SyncError.NoServiceAccount, ) # 3. Validate application ID if sync.application_id != account.application_id: raise PlaylistSynchronizationException( f"Service account app ({account.application_id})" f" does not match sync app ({sync.application_id}).", SyncError.ApplicationMismatch, ) # 4. Fetch source playlist (Spotify) logger.debug( "Fetching source Spotify playlist %s", sync.from_playlist_id ) generic_playlist = _fetch_spotify_source(sync) sync.source_title = generic_playlist.name sync.source_image = generic_playlist.image sync.source_track_count = len(generic_playlist.tracks) sync.source_service_account_id = generic_playlist.user sync.source_service_account_name = generic_playlist.user_name # 5. Route to synchronizer music_service_id = account.music_service_id synchronizer = None if music_service_id == int(MusicService.Spotify): synchronizer = SpotifySynchronizer() elif music_service_id == int(MusicService.Deezer): synchronizer = DeezerSynchronizer() elif music_service_id == int(MusicService.YouTube): synchronizer = YoutubeSynchronizer() elif music_service_id == int(MusicService.SoundCloud): synchronizer = SoundCloudSynchronizer() else: raise PlaylistSynchronizationException( f"Music service {music_service_id} is not supported", SyncError.NotSupported, ) sync_result = await synchronizer.copy_to_playlist( generic_playlist, sync, account, session ) sync.synchronized_track_count = sync_result.synced_track_count sync.last_updated = datetime.now(timezone.utc) sync.error = sync_result.error is not None except Exception as exc: logger.exception("Error during sync %d: %s", sync_id, exc) sync.error = True sync_result = PlaylistSynchronizationResult.from_exception(exc) sync_result.triggered_manually = triggered_manually # 6. Update sync row from playlist_sync.services.sync_service import SyncTaskService task_service = SyncTaskService(session) await task_service.update_sync_result(sync) # 7. Write log row log = PlaylistSynchronizationLog( sync_id=sync.id, time=datetime.now(timezone.utc), made_changes=sync_result.made_changes, added_tracks=sync_result.added_tracks, deleted_tracks=sync_result.deleted_tracks, deleted_duplicates=sync_result.deleted_duplicates, source_tracks=sync.source_track_count or 0, target_track_count=sync_result.synced_track_count, error_message=sync_result.error_text, triggered_manually=sync_result.triggered_manually, error=sync_result.error, # SyncError is a str subclass; str value ) session.add(log) await session.commit() logger.info( "Sync %d done: added=%d removed=%d dupes=%d reordered=%d err=%s", sync_id, sync_result.added_tracks, sync_result.deleted_tracks, sync_result.deleted_duplicates, sync_result.reordered_tracks, sync.error, ) return { "status": "success" if not sync.error else "error", "sync_id": sync_id, "added": sync_result.added_tracks, "removed": sync_result.deleted_tracks, "error": sync_result.error_text, } finally: await async_engine.dispose() try: with sentry_sdk.new_scope() as scope: scope.set_tag("sync_id", sync_id) scope.set_tag("triggered_manually", triggered_manually) return asyncio.run(_run()) except Exception as exc: logger.exception( "Fatal error in execute_single_sync_task for sync_id=%d", sync_id ) raise self.retry(exc=exc, countdown=60) from exc @celery_app.task(name="sync_task.periodic_sweep") def periodic_sync_sweep(): """Periodic sweep — dispatch individual tasks for every active sync. Mirrors: PlaylistSynchronizationTask.ExecuteAsync() Steps: 1. Load all active syncs from DB. 2. Optionally filter by PlaylistSynchronizationByAccountId.txt (one ID per line). 3. Dispatch execute_single_sync_task for each. 4. Re-query after first pass; dispatch any added mid-run or previously failed (second-pass pattern from .NET). Concurrency is controlled via the Celery worker --concurrency flag (mirrors PlaylistSynchronizationTask_NumberOfParallelThreadsToProcessSynchronizations). """ import asyncio from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine from sqlalchemy.orm import sessionmaker as async_sessionmaker from sqlalchemy.pool import NullPool from sqlmodel import select from playlist_sync.models.sync_task import PlaylistSynchronization async def _run(): async_engine = create_async_engine(settings.DATABASE_URL, poolclass=NullPool) try: async_session_factory = async_sessionmaker( async_engine, class_=AsyncSession, expire_on_commit=False ) async with async_session_factory() as session: all_syncs_result = await session.execute( select(PlaylistSynchronization).where( PlaylistSynchronization.active == True ) # noqa: E712 ) all_syncs = list(all_syncs_result.scalars().all()) # Apply optional account ID filter file account_id_filter = _load_account_id_filter() if account_id_filter is not None: all_syncs = [ s for s in all_syncs if s.to_service_account_id in account_id_filter ] logger.info( "periodic_sync_sweep: processing %d active sync(s)", len(all_syncs) ) processed_ids = set() for sync in all_syncs: execute_single_sync_task.delay(sync.id) processed_ids.add(sync.id) # Second pass: re-query to catch syncs added/failed during the first run async with async_session_factory() as session: all_syncs_result = await session.execute( select(PlaylistSynchronization).where( PlaylistSynchronization.active == True ) # noqa: E712 ) second_pass = [ s for s in all_syncs_result.scalars().all() if s.id not in processed_ids ] # Apply the same account filter to the second pass if account_id_filter is not None: second_pass = [ s for s in second_pass if s.to_service_account_id in account_id_filter ] if second_pass: logger.info( "periodic_sync_sweep second pass: %d additional sync(s)", len(second_pass), ) for sync in second_pass: execute_single_sync_task.delay(sync.id) return { "status": "success", "dispatched": len(all_syncs) + len(second_pass), } finally: await async_engine.dispose() return asyncio.run(_run()) def _load_account_id_filter() -> Optional[set[int]]: """Load allowed account IDs from the filter file, if it exists. Mirrors: SynchronizationByAccountIdFilter.GetPredicate() in .NET. Returns None if the file does not exist (meaning no filter is applied). """ path = settings.SYNC_ACCOUNT_FILTER_FILE if not os.path.exists(path): return None with open(path) as f: ids = set() for line in f: line = line.strip() if line: try: ids.add(int(line)) except ValueError: logger.warning("Invalid account ID in filter file: '%s'", line) return ids def _fetch_spotify_source(sync: "PlaylistSynchronization") -> "GenericPlaylist": """Fetch a Spotify playlist and convert to GenericPlaylist. This is the default source fetch path (from_service_type=Spotify). """ from playlist_sync.clients import spotify_client from playlist_sync.models.enums import SyncError from playlist_sync.models.exceptions import PlaylistSynchronizationException from playlist_sync.models.generic_playlist import GenericPlaylist, GenericTrack raw_playlist = spotify_client.get_playlist_with_all_tracks(sync.from_playlist_id) if not raw_playlist: raise PlaylistSynchronizationException( f"Source playlist could not be loaded from Spotify:" f" '{sync.from_playlist_id}'", SyncError.NoSourcePlaylist, ) track_items = raw_playlist.get("tracks", {}).get("items", []) if not track_items: raise PlaylistSynchronizationException( f"Source playlist has no tracks: '{sync.from_playlist_id}'", SyncError.VendorSpecific, ) generic_tracks = [] for item in track_items: if not item or not item.get("track"): continue if item.get("is_local", False): continue t = item["track"] if not t.get("artists"): continue generic_tracks.append( GenericTrack( name=t.get("name", ""), artists=[a["name"] for a in t.get("artists", [])], isrc=t.get("external_ids", {}).get("isrc"), spotify_uri=t.get("uri", ""), ) ) images = raw_playlist.get("images") or [] image_url = images[0].get("url") if images else None return GenericPlaylist( name=raw_playlist.get("name", ""), description=raw_playlist.get("description"), image=image_url, tracks=generic_tracks, user=(raw_playlist.get("owner") or {}).get("id"), user_name=(raw_playlist.get("owner") or {}).get("display_name"), )