"""Tasks for the chartmetric_monitor SWF workflow.""" import backoff import datadog from datadog.api import exceptions from garcon import task from snowflake.connector.errors import ProgrammingError from snowflake_connector.etl_connector import SnowflakeSQLExecutor, SQLLoader from swf_monitoring.flows.chartmetric_monitor import config sqlloader = SQLLoader(__file__) @backoff.on_exception( backoff.expo, (exceptions.ProxyError, exceptions.HTTPError, exceptions.HttpBackoff, exceptions.HttpTimeout)) def _send_event(event): """Send an event using backoff/retry decorator. Args: event (dict): An event to send. """ res = datadog.api.Event.create( title=event['title'], text=event['text'], tags=event['tags'], alert_type='warning') if res.get('status') != 'ok': raise exceptions.HttpBackoff(5) # will be handled by backoff @task.decorate(timeout=60 * 15) def check_tables(activity): """Populate shared_tables_stability table. And send an event to Datadog on each error. """ datadog.initialize(**{'api_key': config.DATADOG_API_KEY}) activity.logger.info('Populating shared_tables_stability table...') for table_name in config.TABLES_TO_MONITOR: params = { 'db': config.SF_CONFIG['db'], 'schema': config.SF_CONFIG['schema'], 'database_name': config.TABLES_TO_MONITOR[table_name]['db'], 'schema_name': config.TABLES_TO_MONITOR[table_name]['schema'], 'table_name': table_name } with SnowflakeSQLExecutor(config.SF_CONFIG) as executor: try: executor.fetchone_query(sqlloader, 'insert_row_count', params) except ProgrammingError: with SnowflakeSQLExecutor( config.SF_CONFIG) as executor: activity.logger.info('Table {} is gone'.format(table_name)) executor.fetchall_query( sqlloader, 'insert_row_count_error', params) event = { 'title': 'CHARTMETRIC shared table ' '{table_name} is gone'.format( table_name=table_name), 'text': 'Table {database_name}.{schema_name}.{table_name} ' 'is not available'.format( table_name=table_name, database_name=config.TABLES_TO_MONITOR[ table_name]['db'], schema_name=config.TABLES_TO_MONITOR[ table_name]['schema']), 'tags': ['chartmetric', 'swf_monitoring'] } _send_event(event) activity.logger.info( 'A warning about table {} sent to Datadog!'.format( table_name)) activity.logger.info('Check for {} finished'.format(table_name)) activity.logger.info('All done!')