"""Garcon tasks related to Jenkins.""" # TODO: move to feed_ingestion.tasks package from garcon import task from garcon_contrib.dynamo_feed_status import garcon_feed_status import jenkins from feed_ingestion.flows.helpers import get_secret from feed_ingestion.tasks import check_status STOP_RESPONSE = {'stop': True} @task.decorate(timeout=600) @check_status() def build_jenkins_dbt(activity, feed_name, licensor, date, build_dbt, config): """Build dbt Jenkins job if data is already ingested. Args: activity (ActivityWorker): The Garcon activity worker. feed_name (str): Feed name. NOT IN USE. REQUIRED FOR DECORATOR!!! licensor (str): Licensor name (sme, theorchard). date (str): Reporting date (YYYY-MM-DD). build_dbt (str, None): If 'True' then run the Jenkins dbt job. """ if build_dbt != 'True': return { 'stop': True, 'reason': 'Skipping, as "build_dbt" is not True' } # we're passing the build_dbt flag explicitly from the Jenkins, e.g.: """ pipenv run python feed_ingestion/bin/exec_flows.py spotify --skip $SKIP --days $DAYS --check-status --context '{"licensor": "theorchard", "build_dbt": "True"}' pipenv run python feed_ingestion/bin/exec_flows.py spotify --skip $SKIP --days $DAYS --check-status --context '{"licensor": "sme"}' """ # noqa # so no need to check the licensor for feed_name_tmpl in config.get('feeds_required_for_jenkins_build'): feed_name = feed_name_tmpl.replace( 'licensorplaceholder', licensor) overall_status = garcon_feed_status.get_overall_status( feed_name, date) if overall_status != garcon_feed_status.STATUS_INGESTED: return { 'stop': True, 'reason': f'{feed_name} at {date} is not ingested yet' } jenkins_url = config.get('jenkins_url') server = jenkins.Jenkins( jenkins_url, username=config.get('jenkins_username'), password=get_secret( config.get('jenkins_secrets_path'), 'JENKINS_API_TOKEN') ) jenkins_job = config.get('jenkins_job') next_build_number = server.get_job_info(jenkins_job)['nextBuildNumber'] queue_id = server.build_job( jenkins_job, config.get('jenkins_job_params')) job_url = f'{jenkins_url}/job/{jenkins_job}/' activity.logger.info( f'{jenkins_job} was triggered. ' f'Build: {next_build_number} - licensor: {licensor} - date: {date}') return { 'message': f'Triggered job {jenkins_job}, queue id: {queue_id}', 'job_url': job_url, }