""" Base Feed Sender Work Flow. Base Work Flow class that new Feed Sender flows should extend. Standardizes how flow domain, name, and workflow_id are generated and adds default on_exception method. Flows should override the decider method. """ import datetime import logging import os from garcon import activity from garcon import param from garcon import runner from garcon import task import raven from feed_sender.conf import config # TODO: move to class when we've deprecated non class based Flows PROD_DOMAIN = 'prod_swf_feed_sender' QA_DOMAIN = 'qa_swf_feed_sender' TEST_DOMAIN = 'test' DEV_DOMAIN = 'dev' logger = logging.getLogger('garcon') class FlowBase(object): """Base Garcon Workflow that other Flows can extend.""" def __init__(self, feed_name, version): """Initialize Flow. Initializes the Flow's: domain - 'prod_swf_feed_ingestion' or 'dev name - '_feed_ingestion' ex. itunes_samis_smart Args: feed_name (str): Name of the feed the work flow is processing, with underscores replacing spaces. ex. 'itunes_samis_smart'. version (str): Version of the workflow type to filter on. should be a 'x.x' float (ex. 1.0, 1.1, etc.) """ self.domain = get_domain() self.name = generate_feed_name(feed_name) self.version = version self.create = activity.create( self.domain, self.name, version=self.version, on_exception=self.on_exception) @staticmethod def get_kwargs_namespace_for(activity_name): """Get kwargs namespace for specified activity. If activity defines kwargs it needs to define if within namespace returned by this function. Args: activity_name (str): name of the activity. Returns: str: namespace. """ return activity_name + '_kwargs' def create_kwargs_collect_activity_for(self, activity_name, mapping): """Create activity that collects values specified by mapping. Usage: @property def collect_kwargs_activity(self): return self.create_kwargs_collect_activity_for( activity_name='load_staging_raw_table', mapping=dict( source_file_name='bootstrap.source_file_name', source_file_size='move_raw_file.source_file_size' ) ) Args: activity_name (str): name of the activity accepting kwargs. mapping (dict): namespace key mapping. Returns: Activity: activity object. """ class NamespaceMappingParam(param.BaseParam): def __init__(self, **mapping): super().__init__() self.mapping = mapping @property def requirements(self): """Return all reqs defined in mapping.""" yield from self.mapping.values() def get_data(self, context): """Resolve and return values for keys from mapping.""" return { k: context.get(v, None) for k, v in self.mapping.items() } @task.decorate(timeout=300) def collect_arguments(activity, kwargs): """Collect kwargs.""" return { 'kwargs': dict(kwargs) } return self.create( name=self.get_kwargs_namespace_for(activity_name), tasks=runner.Sync( collect_arguments.fill( namespace=self.get_kwargs_namespace_for(activity_name), kwargs=NamespaceMappingParam(**mapping) ) ) ) def on_exception(self, actor, exception): """Capture an exception that has occurred in the application. Args: actor (ActivityWorker, DeciderWorker): The actor that has received the exception. exception (Exception): The exception to capture. """ # client grabs sentry dns from SENTRY_DSN environment variable if os.environ.get('SENTRY_DSN'): client = raven.Client() client.captureException() if isinstance(actor, activity.Activity): actor.logger.error(exception) else: logger.error(exception) def workflow_id(self, initial_context=None): """Generate workflow id. Assumes a unique workflow is defined by it's context date and defaults to today's date. If that is not the case this method should be overridden. Args: initial_context (dict): The initial context for the flow. Returns: str: A unique identifier for a workflow being executed In the forms of '-YYYY-MM-DD', where YYYY-MM-DD is the context date or, if none passed the current date. """ if 'context_date' not in initial_context: date = datetime.datetime.today().strftime('%Y-%m-%d') else: date = initial_context['context_date'] return '{flow_name}-{date}'.format( flow_name=self.name, date=date) def decider(self, schedule): """Ensure the workflows will implement the decider method.""" raise NotImplementedError() def generate_feed_name(feed_name): """Generate workflow name. To play nice with non-class based Flows. Args: feed_name (str): Name of the feed the work flow is processing, must be lower case and have spaces replaced with underscores. ex. 'vudu', 'itunes_fanclub' Returns: str: SWF name in the format of '_feed_sender'. """ assert feed_name.islower() and ' ' not in feed_name, ( 'feed name must be lower case with no spaces') return '{feed_name}_feed_sender'.format(feed_name=feed_name) def get_domain(): """Get workflow domain. To play nice with non-class based Flows. Returns: str: SWF domain the workflow should use. """ environment = config.ENV if environment == 'prod': return PROD_DOMAIN if environment == 'qa': return QA_DOMAIN if environment == 'test': return TEST_DOMAIN return DEV_DOMAIN