""" Base Feed Sender Workflow. Base Workflow class that new Feed Sender flows should extend. Standardizes how flow domain, name, and workflow_id are generated and adds default on_exception method. Flows should override the decider method. """ import datetime import logging import os import boto3 from botocore.config import Config from garcon import activity import raven from snowflake_etl.conf import getconf # TODO: move to class when we've deprecated non class based Flows PROD_DOMAIN = 'prod_snowflake_etl' DEV_DOMAIN = 'dev' logger = logging.getLogger('garcon') class FlowBase(object): """Base Garcon Workflow class that other Flows can extend.""" def __init__(self, feed_name=None, version=None, domain=None): """Initialize Flow. Initializes the Flow's: domain - 'prod_swf_feed_ingestion' or 'dev name - '_feed_ingestion' ex. itunes_samis_smart Args: feed_name (str): Name of the feed the work flow is processing, with underscores replacing spaces. ex. 'itunes_samis_smart' version (str): Version of the workflow type to filter on. should be a 'x.x' float (ex. 1.0, 1.1, etc.) domain (str): Optional parameter (used for the flows, triggered from the flow on the different domain). """ self.domain = domain or get_domain() self.name = generate_workflow_name(feed_name) self.version = version or '1.0' self.client = boto3.client('swf', config=Config( connect_timeout=60, region_name=os.environ.get('AWS_REGION', os.environ.get('AWS_DEFAULT_REGION', 'us-east-1')), read_timeout=180, retries={'max_attempts': 2})) self.create = activity.create(self.client, self.domain, self.name, version=self.version, on_exception=self.on_exception) def on_exception(self, actor, exception): """Capture an exception that has occurred in the application. Args: actor (ActivityWorker, DeciderWorker): The actor that has received the exception. exception (Exception): The exception to capture. """ # client grabs Sentry DSN from SENTRY_DSN environment variable if os.environ.get('SENTRY_DSN'): client = raven.Client() client.captureException() if isinstance(actor, activity.Activity): actor.logger.error(exception) else: logger.error(exception) def workflow_id(self, initial_context=None): """Generate workflow id. Assumes a unique workflow is defined by it's context date and defaults to today's date. If that is not the case this method should be overridden. Args: initial_context (dict): The initial context for the flow. Returns: str: A unique identifier for a workflow being executed In the forms of '-YYYY-MM-DD', where YYYY-MM-DD is the context date or, if none passed the current date """ if 'context_date' not in initial_context: date = datetime.datetime.today().strftime('%Y-%m-%d') else: date = initial_context['context_date'] return '{flow_name}-{date}'.format( flow_name=self.name, date=date) def decider(self, schedule): """Ensure the workflows will implement the decider method.""" raise NotImplementedError() def generate_workflow_name(feed_name): """Generate workflow name. To play nice with non-class based Flows. Args: feed_name (str): Name of the feed the work flow is processing, must be lower case and have spaces replaced with underscores. ex. 'vudu', 'itunes_fanclub' Returns: str: SWF name in the format of '_snowflake_etl'. """ assert feed_name.islower() and ' ' not in feed_name, ( 'feed name must be lower case with no spaces') return '{feed_name}_snowflake_etl'.format(feed_name=feed_name) def get_domain(): """Get workflow domain. To play nice with non-class based Flows. Returns: str: SWF domain the workflow should use. """ if getconf('env')['env'] == 'prod': return PROD_DOMAIN else: return DEV_DOMAIN