from celery import Celery from kombu import Exchange, Queue from labelaudit import config from labelaudit.connectors import sentry from labelaudit.consts import label_audit as consts from labelaudit.logic import asset_search from labelaudit.logic import audit_release from labelaudit.logic import release sentry # pep8 app = Celery('label_audit_app') app.config_from_object(dict( BROKER_URL=config.BROKER_URL, CELERY_DEFAULT_QUEUE=config.ASSET_SEARCH_QUEUE, CELERY_ACCEPT_CONTENT=['pickle'], CELERY_QUEUES=(Queue( config.ASSET_SEARCH_QUEUE, Exchange(config.ASSET_SEARCH_QUEUE), routing_key='labelaudit.tasks.asset_search_tasks.#'),), BROKER_TRANSPORT_OPTIONS=dict( queue_name_prefix=config.CELERY_QUEUE_PREFIX))) def process_releases(audit_releases): """Fetches releases from the asset search queue and sends to asset search logic. Args: releases (list): list containing release json objects """ for release_to_process in audit_releases: audit_release_id = release_to_process.get('youtubeAuditReleaseId') release_id = release_to_process.get('releaseId') orchard_my_assets.delay(audit_release_id, release_id) @app.task( name='labelaudit.tasks.asset_search_tasks.orchard_my_assets', queue=config.ASSET_SEARCH_QUEUE, max_retries=config.CELERY_MAX_RETRIES, default_retry_delay=config.CELERY_RETRY_DELAY, acks_late=True) def orchard_my_assets(audit_release_id, release_id): """Searches the YouTube API for assets owned by The Orchard that match the Orchard UPC for the given release. Calls VAPI to fetch the Orchard UPC for this release. Calls VAPI to check for an IODA UPC if YouTube does not return any matching assets. Return: bool: whether or not assets were found on this call """ try: orchard_upc = release.fetch_orchard_upc(release_id).message except Exception as exc: orchard_my_assets.retry(exc=exc) if orchard_upc: account_name = consts.CmsAccounts.get('theorchardmusic').content_owner try: num_assets = asset_search.count_my_assets( orchard_upc, account_name) except Exception as exc: orchard_my_assets.retry(exc=exc) return False if num_assets > 0: audit_params = { 'isCmsAccountAsset': 'Y', 'isYoutubeAsset': 'Y', 'assetCmsAccountName': account_name, 'numAssets': num_assets, 'auditStatus': 'complete'} try: audit_release.update_audit_release( audit_release_id, audit_params) except Exception as exc: orchard_my_assets.retry(exc=exc) return True try: ioda_upc = release.fetch_ioda_upc(release_id).message except Exception as exc: orchard_my_assets.retry(exc=exc) if ioda_upc: ioda_my_assets.delay(audit_release_id, orchard_upc, ioda_upc) else: audit_params = {'isCmsAccountAsset': 'N'} try: audit_release.update_audit_release( audit_release_id, audit_params) except Exception as exc: orchard_my_assets.retry(exc=exc) orchard_all_assets.delay(audit_release_id, orchard_upc, ioda_upc) return False @app.task( name='labelaudit.tasks.asset_search_tasks.ioda_my_assets', queue=config.ASSET_SEARCH_QUEUE, max_retries=config.CELERY_MAX_RETRIES, default_retry_delay=config.CELERY_RETRY_DELAY, acks_late=True) def ioda_my_assets(audit_release_id, orchard_upc, ioda_upc): """Searches the YouTube API for assets owned by IODA that match the given IODA UPC. Return: bool: whether or not assets were found on this call """ account_name = consts.CmsAccounts.get('ioda').content_owner try: num_assets = asset_search.count_my_assets(ioda_upc, account_name) except Exception as exc: ioda_my_assets.retry(exc=exc) return False if num_assets > 0: audit_params = { 'isCmsAccountAsset': 'Y', 'isYoutubeAsset': 'Y', 'assetCmsAccountName': account_name, 'numAssets': num_assets, 'auditStatus': 'complete'} try: audit_release.update_audit_release( audit_release_id, audit_params) except Exception as exc: ioda_my_assets.retry(exc=exc) return True audit_params = {'isCmsAccountAsset': 'N'} try: audit_release.update_audit_release(audit_release_id, audit_params) except Exception as exc: ioda_my_assets.retry(exc=exc) orchard_all_assets.delay(audit_release_id, orchard_upc, ioda_upc) return False @app.task( name='labelaudit.tasks.asset_search_tasks.orchard_all_assets', queue=config.ASSET_SEARCH_QUEUE, max_retries=config.CELERY_MAX_RETRIES, default_retry_delay=config.CELERY_RETRY_DELAY, acks_late=True) def orchard_all_assets(audit_release_id, orchard_upc, ioda_upc): """Searches the YouTube API for any assets that match the given Orchard UPC. Return: bool: whether or not assets were found on this call """ try: search_result = asset_search.search_by_upc( orchard_upc, asset_search.OWNER_ORCHARD, False) except Exception as exc: orchard_all_assets.retry(exc=exc) if search_result.get('items'): num_assets = search_result.get('pageInfo').get('totalResults') audit_params = { 'isYoutubeAsset': 'Y', 'numAssets': num_assets, 'auditStatus': 'complete'} try: audit_release.update_audit_release( audit_release_id, audit_params) except Exception as exc: orchard_all_assets.retry(exc=exc) return True if ioda_upc: ioda_all_assets.delay(audit_release_id, ioda_upc) audit_params = {'isYoutubeAsset': 'N', 'auditStatus': 'complete'} try: audit_release.update_audit_release( audit_release_id, audit_params) except Exception as exc: orchard_all_assets.retry(exc=exc) return False @app.task( name='labelaudit.tasks.asset_search_tasks.ioda_all_assets', queue=config.ASSET_SEARCH_QUEUE, max_retries=config.CELERY_MAX_RETRIES, default_retry_delay=config.CELERY_RETRY_DELAY, acks_late=True) def ioda_all_assets(audit_release_id, ioda_upc): """Searches the YouTube API for any assets that match the given IODA UPC. This is the last possible stage in the asset search logic flow. Return: bool: whether or not assets were found on this call. """ try: search_result = asset_search.search_by_upc( ioda_upc, asset_search.OWNER_IODA, False) except Exception as exc: ioda_all_assets.retry(exc=exc) if search_result.get('items'): num_assets = search_result.get('pageInfo').get('totalResults') audit_params = { 'isYoutubeAsset': 'Y', 'numAssets': num_assets, 'auditStatus': 'complete'} try: audit_release.update_audit_release( audit_release_id, audit_params) except Exception as exc: ioda_all_assets.retry(exc=exc) return True audit_params = {'isYoutubeAsset': 'N', 'auditStatus': 'complete'} try: audit_release.update_audit_release( audit_release_id, audit_params) except Exception as exc: ioda_all_assets.retry(exc=exc) return False