import asyncio import asyncio.runners import logging from functools import partial from app.connectors.database.base import ReportingDB from app.connectors.database.repository import AdReportingRepository from app.models import ( AdReportingCreativeThumbnailUrl, Creative, ) from app.processors import CreativeProcessorsType from app.utils import chunks logger = logging.getLogger(__name__) class AdThumbnailDownloadHandler: def __init__( self, db: ReportingDB, ad_reporting_repository: AdReportingRepository, processors: CreativeProcessorsType, chunk_size: int, threads_count: int, ) -> None: self.db = db self.ad_reporting_repository = ad_reporting_repository self.processors = processors self.chunk_size = chunk_size self.threads_count = threads_count async def handle(self) -> None: logger.info("Ad thumbnails download started") with self.db.session_factory(): creatives = self.ad_reporting_repository.get_creatives() semaphore = asyncio.Semaphore(self.threads_count) logger.info("Creatives to process %s", len(creatives)) processed = 0 for creatives_chunk in chunks(creatives, chunk_size=self.chunk_size): results = await asyncio.gather( *map(partial(self._process_creative, semaphore), creatives_chunk) ) with self.db.session_factory(): self.ad_reporting_repository.store_ad_reporting_creative_thumbnail_urls( results, ) processed += len(creatives_chunk) logger.info("Processed %s/%s creatives", processed, len(creatives)) logger.info("Ad thumbnails download finished") async def _process_creative( self, semaphore: asyncio.Semaphore, creative: Creative ) -> AdReportingCreativeThumbnailUrl: processor = self.processors[creative.platform] async with semaphore: result = await processor.process(creative) return result