""" Transient and self-healing exceptions. We don't want to spam Slack via Sentry with non-actionable stuff. If new non-actionable errors will arise, we should add them to the SEND_AS_WARNINGS list. """ from collections import namedtuple import logging import os import time import sentry_sdk from sentry_sdk.integrations.logging import LoggingIntegration from sentry_sdk.types import Breadcrumb, BreadcrumbHint from feed_ingestion.conf.config import AWS_REGION from feed_ingestion.util import app_version from feed_ingestion.util.aws.swf import generate_execution_console_url logger = logging.getLogger(__name__) DO_NOT_SEND = [ # non-actionable decider error - happened when run was stopped/terminated ('Exception', 'The activity failures has exceeded its retry limit'), # non-actionable worker error - happened when run was stopped/terminated ('UnknownResourceFault', 'when calling the RespondActivityTaskFailed ' 'operation: Unknown execution: ' 'WorkflowExecution'), ('UnknownResourceFault', 'when calling the RespondActivityTaskCompleted ' 'operation: Unknown execution: ' 'WorkflowExecution'), # non-actionable self-fixing errors from youtube_facts, youtube_claims ('ValueError', 'Some dependant reports are not ready:'), ('ValueError', 'No source files found in s3://cucumbers/Youtube_claims'), # NOTE: adding exceptions to this list considered as antipattern # and should be avoided. # It is better to capture exception and handle it where it happens. ] SEND_AS_WARNINGS = [ # non-actionable FTP errors, heals themselves ('SSHException', '[Errno 110] Connection timed out'), ('SSHException', 'Error reading SSH protocol banner'), ('OSError', 'Failure'), # non-actionable Snowflake table lock, heals itself ('ProgrammingError', 'has locked table'), # transient errors on Apple's side, heals itself ('ReporterException', 'Network is available but cannot connect to application'), ('ReporterException', 'Report is unexpectedly not available'), # transient SSL error, heals itself ('timeout', 'The read operation timed out'), # transient socket error, heals itself ('gaierror', '[Errno -2] Name or service not known'), # Spotify API temporarily unavailable ('HTTPError', 'Bad Gateway for url: https://ws.spotify.com/analytics/api/'), # not all the required files were found, retry later will help ('FileNotFoundError', 'Could not find all required files'), # Neo4j re-election process causing running queries to fail ('Exception', 'No write operations are allowed directly on this database'), # Spotify API temporarily unavailable ('HTTPError', '504 Server Error: Gateway Timeout ' 'for url: https://provider-api.spotify.com'), # Spotify API temporarily unavailable ('HTTPError', '503 Server Error: Service Unavailable ' 'for url: https://provider-api.spotify.com'), # Alert regarding exceeded error limit if Spotify API unavailable ('Exception', 'Exception: Load into temp_staging_raw_spotify'), # This exception is caused by exception above ('SWFResponseError', 'SWFResponseError: 400 Bad Request'), ] exception_namedtuple = namedtuple( 'exc', ['exception_classname', 'exception_message']) EXCEPTIONS = [exception_namedtuple(*exc) for exc in SEND_AS_WARNINGS] EXCEPTIONS_NOT_TO_SEND = [exception_namedtuple(*exc) for exc in DO_NOT_SEND] class TTLDict: """Dictionary with time-to-live (TTL) for each key.""" def __init__(self, ttl_seconds): """Initialize a TTL dictionary.""" self._store = {} self.ttl = ttl_seconds def __setitem__(self, key, value): """Set item with key and value, store expiration time.""" expire_at = time.time() + self.ttl self._store[key] = (value, expire_at) def __getitem__(self, key): """Get item by key, raise KeyError if not found or expired.""" value, expire_at = self._store[key] if time.time() > expire_at: del self._store[key] raise KeyError(f'{key} has expired') return value def __contains__(self, key): """Check if key exists and is not expired.""" try: self.__getitem__(key) return True except KeyError: return False def get(self, key, default=None): """Get item by key, return default if not found or expired.""" try: return self.__getitem__(key) except KeyError: return default def clear(self): """Remove all items.""" self._store.clear() def cleanup(self): """Remove all expired items.""" now = time.time() expired_keys = [k for k, (_, exp) in self._store.items() if now > exp] for k in expired_keys: del self._store[k] # we limit the frequency of Sentry events to avoid spamming recently_sent_to_sentry = TTLDict(ttl_seconds=60) def _is_send_as_warning(exception_classname, exception_message): """Check if we want to send an Exception to Sentry as a warning. Args: exception (Exception): An instance of some exception. Returns: bool: True if it's non-actionable, False otherwise. """ for exc in EXCEPTIONS: if (exception_classname.endswith(exc.exception_classname) and exc.exception_message in exception_message): return True return False def _is_not_to_send(exception_classname, exception_message): """Check if we do not want to send an Exception to Sentry. Nor as a warning, neither as an exception. Args: exception (Exception): An instance of some exception. Returns: bool: True if it's non-actionable, False otherwise. """ for exc in EXCEPTIONS_NOT_TO_SEND: if (exception_classname.endswith(exc.exception_classname) and exc.exception_message in exception_message): return True return False def send_message(message, level='error'): """Send message to Sentry.""" dsn = os.environ.get('SENTRY_DSN') if not dsn: return sentry_sdk.capture_message( message, level=level ) def send_error_or_warning(exception, force_send_as_warning=False): """Send exception as error or warning to Sentry. In case the exception is mentioned in DO_NOT_SEND list it won't be sent to Sentry. (_is_not_to_send()) Args: exception (Exception): An instance of some exception. force_send_as_warning (bool): If True, send as warning anyway. """ dsn = os.environ.get('SENTRY_DSN') if not dsn: return if force_send_as_warning: level = 'warning' else: level = 'error' sentry_sdk.capture_exception(exception, level=level) def before_breadcrumb( crumb: Breadcrumb, hint: BreadcrumbHint) -> Breadcrumb | None: """Filter breadcrumbs before sending them to Sentry. * filter out requests to https://swf.*.amazonaws.com """ data = crumb.get('data', {}) if crumb.get('type') == 'http' and ( data.get('url', '').startswith('https://swf.') or data.get('aws.request.url', '').startswith('https://swf.') ): return None return crumb def before_send(event, hint): """Filter and limit the frequency of Sentry events. This function supposed to be used as before_send hook. """ event_str_key = None if 'exception' in event: # take the most outer exception from the stack exception_classname = event['exception']['values'][-1]['type'] exception_message = event['exception']['values'][-1]['value'] event_str_key = f'{exception_classname}: {exception_message}' if _is_not_to_send(exception_classname, exception_message): return None if _is_send_as_warning(exception_classname, exception_message): event['level'] = 'warning' elif 'logentry' in event: # If the event is a log entry, we use the message as the key event_str_key = event['logentry']['formatted'] elif 'message' in event: # If the event is a message, we use the message itself as the key event_str_key = event['message'] # now let's deal with the frequency limit if event_str_key is None: # did not detect event type, ignore frequency limit, pass the event return event if event_str_key in recently_sent_to_sentry: return None recently_sent_to_sentry[event_str_key] = True recently_sent_to_sentry.cleanup() return event def configure_sentry(): """Configure Sentry SDK.""" dsn = os.environ.get('SENTRY_DSN') if not dsn: logger.info('SENTRY_DSN is not set, Sentry will not be configured.') return env_name = os.environ.get('Environment') version = app_version.get_app_version() sentry_sdk.init( dsn=dsn, release=version, environment=env_name, traces_sample_rate=1.0, before_send=before_send, before_breadcrumb=before_breadcrumb, send_default_pii=True, max_breadcrumbs=50, integrations=[ # LoggingIntegration is enabled by default so we customize it LoggingIntegration( # Capture this level and above as breadcrumbs level=logging.WARNING, # do not create sentry issues on log error events event_level=None, # do not capture logs. We use datadog instead sentry_logs_level=None, ), ] ) sentry_sdk.set_tag('git_hash', version) sentry_sdk.set_tag('environment', env_name) logger.info( f'Configured Sentry SDK. env={env_name}, ' f'app_version={version}') class SentryRunnerMixin: """Sentry Runner.""" def execute(self, activity, context): """Execute the activity with Sentry scope.""" tags = {} # add SWF Execution information attribute_map = { 'execution.domain': 'swf_domain', 'execution.run_id': 'swf_run_id', 'execution.workflow_id': 'swf_workflow_id', } for context_key, attribute_key in attribute_map.items(): if context_key not in context: continue tags[attribute_key] = context[context_key] swf_url = generate_execution_console_url( swf_domain=tags.get('swf_domain', ''), workflow_id=tags.get('swf_workflow_id', ''), run_id=tags.get('swf_run_id', ''), aws_region=AWS_REGION) tags['swf_url'] = swf_url with sentry_sdk.isolation_scope() as scope: try: for key, value in tags.items(): scope.set_tag(key, value) scope.set_context('SWF Context', context) scope.set_extra('SWF Extra', tags) return super().execute(activity, context) except Exception as e: # we would use FlowBase.on_exception handler # but activity.run() doesn't provide SWF run info send_error_or_warning(e) raise