"""Tasks of Source of Stream pre-aggregation workflow.""" from garcon import task from analytics_aggregation import base_config from analytics_aggregation.flows.amazon_unlimited_sos.snowflake_executor \ import AmazonUnlimitedSF from analytics_aggregation.util import common as common_utils @task.decorate(timeout=300) def bootstrap( activity, feed_name, source_feed_name, context_date_range=None, days_back=7, reload=False, labelids=None): """Extract date either from passed context, either from ETLs' statuses. Args: activity (ActivityWorker): The activity worker. feed_name (str): Name of the feed. source_feed_name: (str): Name of source feed. context_date_range (str): Date range from context (context_date). days_back (int): Number of last days for check. reload (bool): Flag to force reload and ignore all the statuses. labelids (list | str | int): List of label ids. Optional. If is not None then the whole flow will be run only for these label ids. Can be list of integers or comma-separated ids or single int value. """ try: reload = common_utils.get_bool_from_flag(reload) common_utils.validate_date_range_reload(context_date_range, reload) start_date, end_date = common_utils.find_flow_date_range( feed_name, source_feed_name, context_date_range, days_back) labelids = common_utils.format_id_list(labelids) context = { 'reload': reload, 'date_range_as_str': start_date + '_' + end_date, 'date_range': { 'start_date': start_date, 'end_date': end_date}, 'labelids': labelids} activity.logger.info( 'Start the workflow for {start_date} - {end_date}'.format( start_date=start_date, end_date=end_date)) return context except common_utils.UnprocessableFlowParamsException as error: return common_utils.exit_message(str(error)) @task.decorate(timeout=3600) def cleanup_staging_sos(activity, date_range, labelids): """Delete Amazon data from staging_sos table. Args: date_range (dict): Dict with 2 keys start_date and end_date. labelids (list[int]): List of label ids. """ with AmazonUnlimitedSF(base_config.SNOWFLAKE_CONFIG) as executor: executor.cleanup_staging_sos(date_range, labelids) activity.logger.info('staging_sos table was cleared up.') @task.decorate(timeout=18000) def populate_staging_sos(activity, date_range, labelids): """Populate staging_sos table with Amazon data. Args: date_range (dict): Dict with 2 keys start_date and end_date. labelids (list[int]): List of label ids. """ with AmazonUnlimitedSF(base_config.SNOWFLAKE_CONFIG) as executor: executor.populate_staging_sos(date_range, labelids) activity.logger.info('staging_sos table was populated.')