"""Configuration and marshaling of sql2sf statuses.""" import copy from flask_restful import fields from flask_restful import marshal_with from flask_restful import Resource from feed_status import sql2sf_config as config from feed_status.models.feed_monitor import FeedMonitor from feed_status.models.sql2sf_feed import get_table_statuses from feed_status.models.sql2sf_feed import STATUS_COMPLETED from feed_status.util import generate_default_date_dict_range snowflake_sync_fields = { 'data': fields.List( fields.Nested( { 'schema_name': fields.String, 'tables': fields.List( fields.Nested({ 'table_name': fields.String, 'status': fields.String, 'last_sync_timestamp': fields.DateTime('iso8601'), 'synced_to_timestamp': fields.DateTime('iso8601'), 'metrics': fields.List( fields.Nested({ 'name': fields.String, 'only_business_days': fields.Boolean, 'status': fields.String, 'threshold': fields.Integer }) ) }) ) } ) ) } class SnowflakeSync(Resource): """Provides response for /snowflake_sync//status.""" @marshal_with(snowflake_sync_fields) def get(self, table_descriptor): """Marshal Snowflake synchronization statuses. Args: table_descriptor (str): specific table of interest. `all` - fetch all statuses. Returns: dict: json payload for sync statuses. """ def compute_freshness_metric(tbl_data, tbl_config): """Compute freshness metric for a given table. Args: tbl_data (dict): sync status as returned from DynamoDB. tbl_config (dict): defined params such as threshold, etc. Returns: dict: computed metric along with meta formatted with respect to marshal_with scheme. """ freshness_threshold = tbl_config['freshness_threshold'] only_business_days = tbl_config['only_business_days'] # dynamically create FeedMonitor monitor = FeedMonitor( feed_id='phony', metric_name=config.FRESHNESS_METRIC_NAME, threshold=freshness_threshold, only_business_days=only_business_days, success_status=STATUS_COMPLETED, threshold_time='00:00' ) last_sync_date = tbl_data['last_sync_timestamp'].date() series = generate_default_date_dict_range( from_date=last_sync_date, to_date=last_sync_date) series[last_sync_date.strftime('%Y-%m-%d')] = tbl_data['status'] health = monitor.metric_eval(series) return { 'name': config.FRESHNESS_METRIC_NAME, 'only_business_days': only_business_days, 'status': health, 'threshold': freshness_threshold } assert table_descriptor == 'all', ( 'Table selector is not supported yet (use all)') sources = config.SOURCES target_tables = [ (schema['schema_name'], table['table_name']) for schema in sources for table in schema['tables'] ] statuses = get_table_statuses(target_tables) statuses_copy = copy.deepcopy(statuses) for schema_index, schema in enumerate(statuses_copy): for table_index, table in enumerate(schema['tables']): table_config = config.get_table_config( schema['schema_name'], table['table_name']) statuses[schema_index]['tables'][table_index]['metrics'] = [ compute_freshness_metric(table, table_config) ] return { 'data': statuses }