#! /usr/bin/env python """Helper script to launch a feed ingestion workflow for multiple days.""" import argparse import datetime import json import subprocess from garcon_contrib.dynamo_feed_status import garcon_feed_status from feed_ingestion import flows from feed_ingestion.util import task_status from feed_ingestion.util.aws import swf as swf_util # for mocking in unit tests log = print def exec_multiple_days( flow, days, skip, max_concurrent_flows, dry_run=False, context=None, check_status=False, expected_status=garcon_feed_status.STATUS_INGESTED): """Execute SWF Ingestion Flow for range of days. Will execute a workflow for last N days. The parameters describe the slice of days going backwards from today. So if skip=1 and days=5, it skips today and works on the 5 days before that. Args: flow (str): Name of flow. days (int): Number of past days to run workflow for. skip (int): Number of past days to skip starting from today. max_concurrent_flows (int): Number of flows that can be run at once. dry_run (bool): Output command statements without running. check_status (bool): If True, only exec flow if state is not completed expected_status (str): status value considered as completed """ # since days can include current day, subtract 1 in the offset. This # prevents offsets from always being at least one day back. days_offset = datetime.timedelta(days=days - 1) skip_offset = datetime.timedelta(days=skip) date_N_days_ago = datetime.date.today() date_N_days_ago -= days_offset date_N_days_ago -= skip_offset context = context or '{}' context = json.loads(context) flow_entity = flows.get_flow(flow) num_flows = swf_util.count_running_workflows_by_type( flow_entity.domain, flow_entity.name) for i in range(days): if num_flows >= max_concurrent_flows: break context_date = date_N_days_ago + datetime.timedelta(days=i) context['context_date'] = context_date.strftime('%Y-%m-%d') if check_status: status = False is_completed = True item = garcon_feed_status._get_item( flow_entity.contextified_feed_name(context), context['context_date']) if item and 'status' in item: status = item['status'] is_completed = item.get(task_status.FIELD_COMPLETED, True) if status == expected_status and is_completed: log( '{} feed status for {} is: {}, is completed: {}'.format( flow_entity.contextified_feed_name(context), context['context_date'], status, is_completed)) continue if dry_run: context_string = "'{context}'".format( context=json.dumps(context, sort_keys=True)) call = ['garcon', 'exec', flow, '--context', context_string] log(' '.join(call)) else: context_string = '{context}'.format( context=json.dumps(context, sort_keys=True)) call = ['garcon', 'exec', flow, '--context', context_string] subprocess.call(call) num_flows += 1 if __name__ == '__main__': parser = argparse.ArgumentParser(description='Date Range Flow Exec') parser.add_argument('flow', help='name of the workflow') parser.add_argument( '--days', type=int, default=5, help=( 'number of days to sync in the db from the start date defined by ' 'the skip flag (Default: 5)')) parser.add_argument( '--skip', type=int, default=1, help=( 'number of days backwards from today to skip in the sync ' '(Default: 1)')) parser.add_argument( '--max_concurrent_flows', type=int, default=5, help=( 'number of flows that can be run at once (Default: 5)')) parser.add_argument( '--check-status', action='store_true', help='Only run workflow for days not ingested') parser.add_argument( '--expected-status', type=str, default=garcon_feed_status.STATUS_INGESTED, help=( 'Status value when flow considered as completed' '(Default: INGESTED)')) parser.add_argument( '--dry-run', action='store_true', help='Output the calls without running them') parser.add_argument('--context', help='Shared initial context') args = parser.parse_args() exec_multiple_days( args.flow, args.days, args.skip, args.max_concurrent_flows, args.dry_run, args.context, args.check_status, args.expected_status)