"""Youtube Weekly Data Ingestion Workflow.""" import calendar import datetime from garcon import task from garcon_contrib.dynamo_feed_status import garcon_feed_status from feed_ingestion.flows.youtube_weekly import config from feed_ingestion.tasks import check_ingested_status from feed_ingestion.util import dates_util from feed_ingestion.util import youtube_util STOP_RESPONSE = {'stop': True} @task.decorate(timeout=1000) @check_ingested_status(config.feed_name) def bootstrap(activity, date, dw_config=None): """Bootstrap workflow by injecting initial context from config. Args: activity (ActivityWorker): The Garcon activity worker. date (str): Reporting date (YYYY-MM-DD). dw_config (dict): Dictionary of data warehouse config options. Returns: dict: Initial context for the workflow. """ date_obj, start_date, end_date = youtube_util.get_days(date) s3_archive_path = 's3://{bucket}/{s3_path}'.format( bucket=config.data_bucket, s3_path=config.s3['archive_path'].format(date=date_obj)) s3_download_path = 's3://{bucket}/{s3_path}'.format( bucket=config.drop_bucket, s3_path=config.s3['download_path']) source_files_dict = { 'files': [{ 'file_name': config.file_template.format( file_type=file_type, date=date_obj), } for file_type in config.file_types]} return { 'feed_name': config.feed_name, 'secrets_path': config.secrets_path, 'date': date, 's3_archive_path': s3_archive_path, 's3_download_path': s3_download_path, 'source_files_dict': source_files_dict, 'staging_raw_table': config.staging_raw_table_name} @task.decorate(timeout=1000) def check_feed_status(activity, date, reload): """Check and reset feed status if it is needed. Args: activity (ActivityWorker): The Garcon activity worker. date (str): Reporting date (YYYY-MM-DD). reload (str or None): If 'True' delete all feed statuses in DynamoDB. Returns: dict: {} or STOP_RESPONSE from check_ingested_status decorator """ date_obj = datetime.datetime.strptime(date, '%Y-%m-%d') report_date = dates_util.get_first_weekday_before_date( date_obj, calendar.MONDAY).strftime('%Y-%m-%d') if reload == 'True': activity.logger.info('Delete status for feed: {} {} '.format( config.feed_name, report_date)) garcon_feed_status.delete_status(config.feed_name, report_date) else: overall_status = garcon_feed_status.get_overall_status( config.feed_name, report_date) if overall_status == garcon_feed_status.STATUS_INGESTED: return STOP_RESPONSE return {'date': report_date}