"""Service level flow classes.""" import json import os from garcon import activity from garcon import param import raven from flows import datastore class FlowBase: """Base class for Garcon Flow.""" def __init__(self, domain, workflow_name, workflow_version): """Initialize Flow.""" self.domain = domain self.name = workflow_name self.version = workflow_version self.create = activity.create( self.domain, self.name, version=self.version, on_exception=self.on_exception) def on_exception(self, actor, exception): """Capture an exception that has occurred in the application. Args: actor (ActivityWorker, DeciderWorker): the actor that has received the exception. exception (Exception): the exception to capture. """ print(exception) # client grabs sentry dns from SENTRY_DSN environment variable if os.environ.get('SENTRY_DSN'): client = raven.Client() client.captureException() pass if isinstance(actor, activity.Activity): actor.logger.error(exception) class DatabaseParam(param.BaseParam): """Task parameter wrapper to get data from datastore instead of SWF. To get around SWF's context size limitations, this class allows the data to be centralized in a third party location. In film transparency it is set to live in the datastore. The initialized context_key is similar to what is used in garcon tasks. Depending on the task filled param types, garcon retrieves the data differently. Example: some_task.fill( correlation_id='correlation_id', some_data=StaticParam(config.SOME_STATIC_DATA), upcs=DatabaseParam('upcs')) Going through the params values (not keys): correlation_id: is a string type, so it is the key lookup in the SWF's context json data. Logically the same as swf_context['correlation_id']. some_data: is a StaticParam object, so the initialized value is directly passed to the task call. upcs: is this DatabaseParam object. The initialized value and the correlation_id value are both required. It uses both to do an SQL query to get the full data to pass to the task call. """ def __init__(self, context_key, data=None): """Initialize the database param object. Args: context_key (str): unique key to this ETL based on correlation ID. data (any): optional data object to initialize with. """ self.context_key = context_key self.data = data self.correlation_id = None self._data_json = None @property def data_json(self): """Convenience getter to get the JSON encoded data. Returns: str: JSON encoded data. """ if self._data_json is None: self._data_json = json.dumps(self.data) return self._data_json @property def requirements(self): """Garcon level dependencies. This ensures the correlation_id key also exists in the task. Yields: str: different keys required in the SWF context. """ yield self.context_key yield 'correlation_id' def get_data(self, context): """Get data for a task. Args: context (dict): existing garcon context data to refer to. Returns: object: JSON decoded data from datastore lookup. """ current_correlation_id = context['correlation_id'].split('.')[0] if self.data is None or self.correlation_id != current_correlation_id: self.correlation_id = current_correlation_id self.data = self.get_data_from_db(self.correlation_id, self.context_key) return self.data def get_data_from_db(self, correlation_id, context_key): """Get data from the datastore. Args: correlation_id (str): ETL correlation ID. context_key (str): ETL specific unique key. Returns: object: JSON decoded data from datastore lookup. """ sql = """ SELECT data FROM etl_context WHERE correlation_id = %s AND context_key = %s;""" try: results = datastore.query(sql, (correlation_id, context_key)) row = results.fetchone() return json.loads(row[0]) except: return None def put_data(self, correlation_id): """Put data into the datastore to be retrieved later in other tasks. This is not a default garcon hook, but is included in this class as a consistent way to ensure the data is where it needs to be. This should be used with SWF execution creation logic. Args: correlation_id (str): ETL correlation ID. """ sql = """ INSERT INTO etl_context (correlation_id, context_key, data) VALUES (%s, %s, %s) ON DUPLICATE KEY UPDATE data = %s;""" params = ( correlation_id.split('.')[0], self.context_key, self.data_json, self.data_json) datastore.execute(sql, params) def __eq__(self, other): """Validate values are the same across params. Args: other (any): object to compare to. Returns: bool: True if it is the same class with same context_key and data. """ return self.data == other.data and \ self.context_key == other.context_key